diff --git a/src/Microsoft.IdentityModel.Protocols/Configuration/ConfigurationManager.cs b/src/Microsoft.IdentityModel.Protocols/Configuration/ConfigurationManager.cs
index b5e6bf0eba..b685c0bc95 100644
--- a/src/Microsoft.IdentityModel.Protocols/Configuration/ConfigurationManager.cs
+++ b/src/Microsoft.IdentityModel.Protocols/Configuration/ConfigurationManager.cs
@@ -248,22 +248,26 @@ public virtual async Task<T> GetConfigurationAsync(CancellationToken cancel)
                 {
                     if (_refreshRequested)
                     {
+                        _refreshRequested = false;
                         // Log as manual because RequestRefresh was called
                         TelemetryClient.IncrementConfigurationRefreshRequestCounter(
                             MetadataAddress,
                             TelemetryConstants.Protocols.Manual);
 
                         UpdateCurrentConfiguration();
-                        _refreshRequested = false;
                     }
-                    else
+                    else if (SyncAfter <= _timeProvider.GetUtcNow())
                     {
                         TelemetryClient.IncrementConfigurationRefreshRequestCounter(
-                            MetadataAddress,
-                            TelemetryConstants.Protocols.Automatic);
+                        MetadataAddress,
+                        TelemetryConstants.Protocols.Automatic);
 
                         _ = Task.Run(UpdateCurrentConfiguration, CancellationToken.None);
                     }
+                    else
+                    {
+                        Interlocked.Exchange(ref _configurationRetrieverState, ConfigurationRetrieverIdle);
+                    }
                 }
             }