diff --git a/sdk/spring/spring-cloud-azure-autoconfigure/src/main/java/com/azure/spring/cloud/autoconfigure/implementation/aad/configuration/AadResourceServerConfiguration.java b/sdk/spring/spring-cloud-azure-autoconfigure/src/main/java/com/azure/spring/cloud/autoconfigure/implementation/aad/configuration/AadResourceServerConfiguration.java
index 8eaa7a62d71e..c5d32f0c476a 100644
--- a/sdk/spring/spring-cloud-azure-autoconfigure/src/main/java/com/azure/spring/cloud/autoconfigure/implementation/aad/configuration/AadResourceServerConfiguration.java
+++ b/sdk/spring/spring-cloud-azure-autoconfigure/src/main/java/com/azure/spring/cloud/autoconfigure/implementation/aad/configuration/AadResourceServerConfiguration.java
@@ -32,6 +32,7 @@
 import org.springframework.security.web.SecurityFilterChain;
 import org.springframework.util.StringUtils;
 
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
@@ -43,6 +44,9 @@
 @Conditional(ResourceServerCondition.class)
 class AadResourceServerConfiguration {
 
+    private static final Duration JWT_DECODER_CONNECT_TIMEOUT = Duration.ofMillis(500);
+    private static final Duration JWT_DECODER_READ_TIMEOUT = Duration.ofMillis(500);
+
     private final RestTemplateBuilder restTemplateBuilder;
 
     AadResourceServerConfiguration(RestTemplateBuilder restTemplateBuilder) {
@@ -57,8 +61,10 @@ JwtDecoder jwtDecoder(AadAuthenticationProperties aadAuthenticationProperties) {
             aadAuthenticationProperties.getProfile().getEnvironment().getActiveDirectoryEndpoint(), tenantId);
         NimbusJwtDecoder nimbusJwtDecoder = NimbusJwtDecoder
             .withJwkSetUri(identityEndpoints.getJwkSetEndpoint())
-                .restOperations(createRestTemplate(restTemplateBuilder))
-                .build();
+            .restOperations(createRestTemplate(restTemplateBuilder
+                .connectTimeout(JWT_DECODER_CONNECT_TIMEOUT)
+                .readTimeout(JWT_DECODER_READ_TIMEOUT)))
+            .build();
         List<OAuth2TokenValidator<Jwt>> validators = createDefaultValidator(aadAuthenticationProperties);
         nimbusJwtDecoder.setJwtValidator(new DelegatingOAuth2TokenValidator<>(validators));
         return nimbusJwtDecoder;