Merge pull request #483 from NuGet/dev

[ReleasePrep][2018.07.16]RI of dev into master

src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs

@@ -1,13 +1,30 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
+
 namespace NuGet.Services.Revalidate
 {
     public class RevalidationConfiguration
     {
+        /// <summary>
+        /// The time before the revalidation job restarts itself.
+        /// </summary>
+        public TimeSpan ShutdownWaitInterval { get; set; } = TimeSpan.FromDays(1);
+
+        /// <summary>
+        /// How long the revalidation job should wait if a revalidation cannot be processed at this time.
+        /// </summary>
+        public TimeSpan RetryLaterSleep { get; set; } = TimeSpan.FromMinutes(5);
+
         /// <summary>
         /// The configurations used to initialize the revalidation state.
         /// </summary>
         public InitializationConfiguration Initialization { get; set; }
+
+        /// <summary>
+        /// The configurations used by the in-memory queue of revalidations to start.
+        /// </summary>
+        public RevalidationQueueConfiguration Queue { get; set; }
     }
 }

src/NuGet.Services.Revalidate/Configuration/RevalidationQueueConfiguration.cs

@@ -0,0 +1,21 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Services.Revalidate
+{
+    public class RevalidationQueueConfiguration
+    {
+        /// <summary>
+        /// The maximum times that the <see cref="RevalidationQueue"/> should look for a revalidation
+        /// before giving up.
+        /// </summary>
+        public int MaximumAttempts { get; set; } = 5;
+
+        /// <summary>
+        /// The time to sleep after an initialized revalidation is deemed completed.
+        /// </summary>
+        public TimeSpan SleepBetweenAttempts { get; set; } = TimeSpan.FromSeconds(5);
+    }
+}

src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs

@@ -189,7 +189,6 @@ private HashSet<int> FindRegistrationKeys(string setName, Expression<Func<Packag
                     .ToList();
 
                 result.UnionWith(batchResults);
-                start = batchResults.Last();
 
                 _logger.LogInformation("Found {Results} results for package set {SetName}", result.Count, setName);
 
@@ -199,6 +198,8 @@ private HashSet<int> FindRegistrationKeys(string setName, Expression<Func<Packag
                 }
                 else
                 {
+                    start = batchResults.Last();
+
                     _logger.LogInformation(
                         "Sleeping for {SleepDuration} before searching for more package set {SetName} results",
                         _config.SleepDurationBetweenBatches,

src/NuGet.Services.Revalidate/Job.cs

@@ -13,6 +13,9 @@
 using NuGet.Jobs;
 using NuGet.Jobs.Configuration;
 using NuGet.Jobs.Validation;
+using NuGet.Services.Logging;
+using NuGet.Services.ServiceBus;
+using NuGet.Services.Validation;
 using NuGetGallery;
 
 namespace NuGet.Services.Revalidate
@@ -26,6 +29,8 @@ public class Job : JsonConfigurationJob
         private const string VerifyInitializationArgumentName = "VerifyInitialization";
         private const string JobConfigurationSectionName = "RevalidateJob";
 
+        private static readonly TimeSpan RetryLaterSleepDuration = TimeSpan.FromMinutes(5);
+
         private bool _initialize;
         private bool _verifyInitialization;
 
@@ -75,17 +80,23 @@ public override async Task Run()
                 }
                 else
                 {
-                    // TODO: https://github.com/NuGet/Engineering/issues/1443
-                    // Send revalidation requests to the Orchestrator.
-                    throw new NotImplementedException();
+                    Logger.LogInformation("Running the revalidation service...");
+
+                    await scope.ServiceProvider
+                        .GetRequiredService<IRevalidationService>()
+                        .RunAsync();
+
+                    Logger.LogInformation("Revalidation service finished running");
                 }
             }
         }
 
         protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
         {
             services.Configure<RevalidationConfiguration>(configurationRoot.GetSection(JobConfigurationSectionName));
+            services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Initialization);
+            services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Queue);
 
             services.AddScoped<IGalleryContext>(provider =>
             {
@@ -94,9 +105,31 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
                 return new GalleryContext(config.ConnectionString, readOnly: false);
             });
 
-            services.AddScoped<IRevalidationStateService, RevalidationStateService>();
-            services.AddScoped<IPackageFinder, PackageFinder>();
-            services.AddScoped<InitializationManager>();
+            // Core
+            services.AddTransient<ITelemetryService, TelemetryService>();
+            services.AddTransient<ITelemetryClient, TelemetryClientWrapper>();
+
+            services.AddTransient<IRevalidationStateService, RevalidationStateService>();
+
+            // Initialization
+            services.AddTransient<IPackageFinder, PackageFinder>();
+            services.AddTransient<InitializationManager>();
+
+            // Revalidation
+            services.AddTransient<IHealthService, HealthService>();
+            services.AddTransient<IRevalidationQueue, RevalidationQueue>();
+            services.AddTransient<IRevalidationService, RevalidationService>();
+            services.AddTransient<IRevalidationThrottler, RevalidationThrottler>();
+            services.AddTransient<ISingletonService, SingletonService>();
+
+            services.AddTransient<IPackageValidationEnqueuer, PackageValidationEnqueuer>();
+            services.AddTransient<IServiceBusMessageSerializer, ServiceBusMessageSerializer>();
+            services.AddTransient<ITopicClient>(provider =>
+            {
+                var config = provider.GetRequiredService<IOptionsSnapshot<ServiceBusConfiguration>>().Value;
+
+                return new TopicClientWrapper(config.ConnectionString, config.TopicPath);
+            });
         }
 
         protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)

src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj

@@ -44,16 +44,28 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Configuration\InitializationConfiguration.cs" />
+    <Compile Include="Configuration\RevalidationQueueConfiguration.cs" />
     <Compile Include="Extensions\IEnumerableExtensions.cs" />
     <Compile Include="Initialization\InitializationManager.cs" />
     <Compile Include="Configuration\RevalidationConfiguration.cs" />
     <Compile Include="Initialization\IPackageFinder.cs" />
     <Compile Include="Initialization\PackageFinder.cs" />
     <Compile Include="Initialization\PackageRegistrationInformation.cs" />
-    <Compile Include="IRevalidationStateService.cs" />
-    <Compile Include="RevalidationStateService.cs" />
-    <Compile Include="TelemetryService.cs" />
-    <Compile Include="ITelemetryService.cs" />
+    <Compile Include="Services\HealthService.cs" />
+    <Compile Include="Services\IHealthService.cs" />
+    <Compile Include="Services\IRevalidationQueue.cs" />
+    <Compile Include="Services\IRevalidationStateService.cs" />
+    <Compile Include="Services\IRevalidationService.cs" />
+    <Compile Include="Services\IRevalidationThrottler.cs" />
+    <Compile Include="Services\ISingletonService.cs" />
+    <Compile Include="Services\RevalidationQueue.cs" />
+    <Compile Include="Services\RevalidationResult.cs" />
+    <Compile Include="Services\RevalidationService.cs" />
+    <Compile Include="Services\RevalidationStateService.cs" />
+    <Compile Include="Services\RevalidationThrottler.cs" />
+    <Compile Include="Services\SingletonService.cs" />
+    <Compile Include="Services\TelemetryService.cs" />
+    <Compile Include="Services\ITelemetryService.cs" />
     <Compile Include="Job.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />

src/NuGet.Services.Revalidate/Services/HealthService.cs

@@ -0,0 +1,17 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Revalidate
+{
+    public class HealthService : IHealthService
+    {
+        public Task<bool> IsHealthyAsync()
+        {
+            // TODO:
+            // We are software gods that never make mistakes.
+            return Task.FromResult(true);
+        }
+    }
+}

src/NuGet.Services.Revalidate/Services/IHealthService.cs

@@ -0,0 +1,16 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Revalidate
+{
+    public interface IHealthService
+    {
+        /// <summary>
+        /// Determine whether the NuGet service is healthy.
+        /// </summary>
+        /// <returns>Whether the NuGet service is healthy.</returns>
+        Task<bool> IsHealthyAsync();
+    }
+}

src/NuGet.Services.Revalidate/Services/IRevalidationQueue.cs

@@ -0,0 +1,17 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+using NuGet.Services.Validation;
+
+namespace NuGet.Services.Revalidate
+{
+    public interface IRevalidationQueue
+    {
+        /// <summary>
+        /// Fetch the next package to revalidate.
+        /// </summary>
+        /// <returns>The next package to revalidate, or null if there are no packages to revalidate at this time.</returns>
+        Task<PackageRevalidation> NextOrNullAsync();
+    }
+}

src/NuGet.Services.Revalidate/ITelemetryService.cs → src/NuGet.Services.Revalidate/Services/IRevalidationService.cs

@@ -1,11 +1,14 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using System;
+using System.Threading.Tasks;
 
 namespace NuGet.Services.Revalidate
 {
-    public interface ITelemetryService
+    public interface IRevalidationService
     {
+        Task RunAsync();
+
+        Task<RevalidationResult> StartNextRevalidationAsync();
     }
 }

src/NuGet.Services.Revalidate/IRevalidationStateService.cs → src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs

@@ -1,7 +1,6 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using NuGet.Services.Validation;
@@ -10,6 +9,12 @@ namespace NuGet.Services.Revalidate
 {
     public interface IRevalidationStateService
     {
+        /// <summary>
+        /// Check whether the killswitch has been activated. If it has, all revalidation operations should be halted.
+        /// </summary>
+        /// <returns>Whether the killswitch has been activated.</returns>
+        Task<bool> IsKillswitchActiveAsync();
+
         /// <summary>
         /// Add the new revalidations to the database.
         /// </summary>
@@ -27,5 +32,12 @@ public interface IRevalidationStateService
         /// </summary>
         /// <returns>The count of package revalidations in the database.</returns>
         Task<int> PackageRevalidationCountAsync();
+
+        /// <summary>
+        /// Update the package revalidation and mark is as enqueued.
+        /// </summary>
+        /// <param name="revalidation">The revalidation to update.</param>
+        /// <returns>A task that completes once the revalidation has been updated.</returns>
+        Task MarkRevalidationAsEnqueuedAsync(PackageRevalidation revalidation);
     }
 }

src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs

@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Revalidate
+{
+    public interface IRevalidationThrottler
+    {
+        /// <summary>
+        /// Check whether the revalidation capacity has been reached.
+        /// </summary>
+        /// <returns>If true, no more revalidations should be performed.</returns>
+        Task<bool> IsThrottledAsync();
+
+        /// <summary>
+        /// Reset the capacity to the configured minimum value. Call this when the service's status is degraded to
+        /// throttle the revalidations.
+        /// </summary>
+        /// <returns>A task that completes once the capacity theshold has been reset.</returns>
+        Task ResetCapacityAsync();
+
+        /// <summary>
+        /// Increase the revalidation capacity by one revalidation per minute.
+        /// </summary>
+        /// <returns>A task taht completes once the capacity has been increased.</returns>
+        Task IncreaseCapacityAsync();
+
+        /// <summary>
+        /// Delay the current task to achieve the desired revalidation rate.
+        /// </summary>
+        /// <returns>Delay the task to ensure the desired revalidation rate.</returns>
+        Task DelayUntilNextRevalidationAsync();
+
+        /// <summary>
+        /// Delay the current task until when a revalidation can be retried.
+        /// </summary>
+        /// <returns>Delay the task until when revalidations can be retried.</returns>
+        Task DelayUntilRevalidationRetryAsync();
+    }
+}

src/NuGet.Services.Revalidate/Services/ISingletonService.cs

@@ -0,0 +1,19 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Revalidate
+{
+    /// <summary>
+    /// Used to ensure that only one instance of this service is running at once.
+    /// </summary>
+    public interface ISingletonService
+    {
+        /// <summary>
+        /// Determines whether this is the only instance of the service running.
+        /// </summary>
+        /// <returns>True if this service is the only instance of the service running.</returns>
+        Task<bool> IsSingletonAsync();
+    }
+}

src/NuGet.Services.Revalidate/Services/ITelemetryService.cs

@@ -0,0 +1,16 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Services.Revalidate
+{
+    public interface ITelemetryService
+    {
+        IDisposable TrackDurationToStartNextRevalidation();
+
+        void TrackPackageRevalidationMarkedAsCompleted(string packageId, string normalizedVersion);
+
+        void TrackPackageRevalidationStarted(string packageId, string normalizedVersion);
+    }
+}