diff --git a/src/ArchivePackages/ArchivePackages.Job.cs b/src/ArchivePackages/ArchivePackages.Job.cs index 97f7528e6..4e036b203 100644 --- a/src/ArchivePackages/ArchivePackages.Job.cs +++ b/src/ArchivePackages/ArchivePackages.Job.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Diagnostics.Tracing; using System.Linq; using System.Threading.Tasks; @@ -12,8 +13,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace ArchivePackages { @@ -53,8 +52,6 @@ public class Job : JobBase /// Blob containing the cursor data. Cursor data comprises of cursorDateTime /// public string CursorBlobName { get; set; } - - private ISqlConnectionFactory _packageDbConnectionFactory; protected CloudBlobContainer SourceContainer { get; private set; } @@ -62,13 +59,13 @@ public class Job : JobBase protected CloudBlobContainer SecondaryDestinationContainer { get; private set; } + private SqlConnectionStringBuilder PackageDatabase { get; set; } + public Job() : base(JobEventSource.Log) { } public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - _packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); + PackageDatabase = RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); Source = CloudStorageAccount.Parse( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.Source)); @@ -92,13 +89,18 @@ public override void Init(IServiceContainer serviceContainer, IDictionary packages; - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) { packages = (await connection.QueryAsync(@" SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited, p.Published @@ -135,7 +138,8 @@ FROM Packages p WHERE Published > @cursorDateTime OR LastEdited > @cursorDateTime", new { cursorDateTime = cursorDateTime })) .ToList(); } - JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + + JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, PackageDatabase.DataSource, PackageDatabase.InitialCatalog); var archiveSet = packages .AsParallel() diff --git a/src/ArchivePackages/ArchivePackages.csproj b/src/ArchivePackages/ArchivePackages.csproj index efb41445d..900001677 100644 --- a/src/ArchivePackages/ArchivePackages.csproj +++ b/src/ArchivePackages/ArchivePackages.csproj @@ -75,9 +75,6 @@ 9.0.1 - - 2.25.0-master-30453 - 4.3.3 diff --git a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj index 696c6c793..5fc084f77 100644 --- a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj +++ b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj @@ -92,9 +92,6 @@ 9.0.1 - - 2.25.0-master-30263 - 2.1.3 diff --git a/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs b/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs index e1021e836..e9db4dc9a 100644 --- a/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs +++ b/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs @@ -6,7 +6,6 @@ using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; -using NuGet.Services.Sql; using Gallery.CredentialExpiration.Models; namespace Gallery.CredentialExpiration @@ -14,12 +13,15 @@ namespace Gallery.CredentialExpiration public class GalleryCredentialExpiration : ICredentialExpirationExporter { private readonly CredentialExpirationJobMetadata _jobMetadata; - private readonly ISqlConnectionFactory _galleryDatabase; - public GalleryCredentialExpiration(CredentialExpirationJobMetadata jobMetadata, ISqlConnectionFactory galleryDatabase) + private Func> OpenGallerySqlConnectionAsync { get; } + + public GalleryCredentialExpiration( + CredentialExpirationJobMetadata jobMetadata, + Func> openGallerySqlConnectionAsync) { _jobMetadata = jobMetadata; - _galleryDatabase = galleryDatabase; + OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync; } /// @@ -51,7 +53,7 @@ public async Task> GetCredentialsAsync(TimeSpan time var minNotificationDate = ConvertToString(GetMinNotificationDate()); // Connect to database - using (var galleryConnection = await _galleryDatabase.CreateAsync()) + using (var galleryConnection = await OpenGallerySqlConnectionAsync()) { // Fetch credentials that expire in _warnDaysBeforeExpiration days // + the user's e-mail address diff --git a/src/Gallery.CredentialExpiration/Job.cs b/src/Gallery.CredentialExpiration/Job.cs index 4290dba1b..e0340eb0a 100644 --- a/src/Gallery.CredentialExpiration/Job.cs +++ b/src/Gallery.CredentialExpiration/Job.cs @@ -2,7 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel.Design; using System.Data.SqlClient; @@ -15,10 +14,7 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using NuGet.Services.Storage; namespace Gallery.CredentialExpiration @@ -34,8 +30,6 @@ public class Job : JobBase private string _galleryBrand; private string _galleryAccountUrl; - private ISqlConnectionFactory _galleryDatabase; - private string _mailFrom; private SmtpClient _smtpClient; @@ -47,9 +41,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenGallerySqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase); + } + public override async Task Run() { var jobRunTime = DateTimeOffset.UtcNow; // Default values var jobCursor = new JobRunTimeCursor( jobCursorTime: jobRunTime, maxProcessedCredentialsTime: jobRunTime ); - var galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase); + var galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), OpenGallerySqlConnectionAsync); try { @@ -89,7 +86,7 @@ public override async Task Run() // Load from cursor // Throw if the schema is not correct to ensure that not-intended emails are sent. jobCursor = JsonConvert.DeserializeObject(content, new JsonSerializerSettings() { MissingMemberHandling = MissingMemberHandling.Error }); - galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase); + galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), OpenGallerySqlConnectionAsync); } // Connect to database diff --git a/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs b/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs index dc0d07ca5..14e4d90d9 100644 --- a/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs +++ b/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using Gallery.Maintenance.Models; using Microsoft.Extensions.Logging; +using NuGet.Jobs; namespace Gallery.Maintenance { @@ -37,7 +38,7 @@ public override async Task RunAsync(Job job) { IEnumerable expiredKeys; - using (var connection = await job.GalleryDatabase.CreateAsync()) + using (var connection = await job.OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase)) { expiredKeys = await connection.QueryWithRetryAsync( SelectQuery, @@ -59,7 +60,7 @@ public override async Task RunAsync(Job job) if (expectedRowCount > 0) { - using (var connection = await job.GalleryDatabase.CreateAsync()) + using (var connection = await job.OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase)) { using (var transaction = connection.BeginTransaction()) { diff --git a/src/Gallery.Maintenance/Gallery.Maintenance.csproj b/src/Gallery.Maintenance/Gallery.Maintenance.csproj index b11f12415..e6a3ed08b 100644 --- a/src/Gallery.Maintenance/Gallery.Maintenance.csproj +++ b/src/Gallery.Maintenance/Gallery.Maintenance.csproj @@ -67,9 +67,6 @@ 9.0.1 - - 2.25.0-master-30263 - 4.3.3 diff --git a/src/Gallery.Maintenance/Job.cs b/src/Gallery.Maintenance/Job.cs index 488a49b92..55d64f291 100644 --- a/src/Gallery.Maintenance/Job.cs +++ b/src/Gallery.Maintenance/Job.cs @@ -4,13 +4,11 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Gallery.Maintenance { @@ -19,15 +17,9 @@ namespace Gallery.Maintenance /// public class Job : JobBase { - - public ISqlConnectionFactory GalleryDatabase { get; private set; } - public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase); - - GalleryDatabase = new AzureSqlConnectionFactory(databaseConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.GalleryDatabase); } public override async Task Run() diff --git a/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs b/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs deleted file mode 100644 index 608481d2c..000000000 --- a/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Threading.Tasks; - -// ReSharper disable once CheckNamespace -namespace System.Data.SqlClient -{ - public static class SqlConnectionStringBuilderExtensions - { - public static Task ConnectTo(this SqlConnectionStringBuilder self) - { - return ConnectTo(self.ConnectionString); - } - - private static async Task ConnectTo(string connection) - { - var c = new SqlConnection(connection); - await c.OpenAsync().ConfigureAwait(continueOnCapturedContext: false); - return c; - } - } -} \ No newline at end of file diff --git a/src/NuGet.Jobs.Common/JobBase.cs b/src/NuGet.Jobs.Common/JobBase.cs index e16184854..b16f92846 100644 --- a/src/NuGet.Jobs.Common/JobBase.cs +++ b/src/NuGet.Jobs.Common/JobBase.cs @@ -1,11 +1,18 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Diagnostics.Tracing; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Jobs.Configuration; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; namespace NuGet.Jobs { @@ -13,6 +20,8 @@ public abstract class JobBase { private readonly EventSource _jobEventSource; + private Dictionary _sqlConnectionFactories; + protected JobBase() : this(null) { @@ -22,6 +31,7 @@ protected JobBase(EventSource jobEventSource) { JobName = GetType().ToString(); _jobEventSource = jobEventSource; + _sqlConnectionFactories = new Dictionary(); } public string JobName { get; private set; } @@ -36,6 +46,132 @@ public void SetLogger(ILoggerFactory loggerFactory, ILogger logger) Logger = logger; } + /// + /// Test connection early to fail fast, and log connection diagnostics. + /// + private async Task TestConnection(ISqlConnectionFactory connectionFactory) + { + try + { + using (var connection = await connectionFactory.OpenAsync()) + using (var cmd = new SqlCommand("SELECT CONCAT(CURRENT_USER, '/', SYSTEM_USER)", connection)) + { + var result = cmd.ExecuteScalar(); + var user = result.ToString(); + Logger.LogInformation("Connected to database {DataSource}/{InitialCatalog} as {User}", + connectionFactory.DataSource, connectionFactory.InitialCatalog, user); + } + } + catch (Exception e) + { + Logger.LogError(0, e, "Failed to connect to database {DataSource}/{InitialCatalog}", + connectionFactory.DataSource, connectionFactory.InitialCatalog); + } + } + + /// + /// Initializes an , for use by validation jobs. + /// + /// ConnectionStringBuilder, used for diagnostics. + public SqlConnectionStringBuilder RegisterDatabase(IServiceProvider serviceProvider) + where T : IDbConfiguration + { + if (serviceProvider == null) + { + throw new ArgumentNullException(nameof(serviceProvider)); + } + + var secretInjector = serviceProvider.GetRequiredService(); + var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; + var connectionFactory = new AzureSqlConnectionFactory(connectionString, secretInjector); + + return RegisterDatabase(nameof(T), connectionString, secretInjector); + } + + /// + /// Initializes an , for use by non-validation jobs. + /// + /// ConnectionStringBuilder, used for diagnostics. + public SqlConnectionStringBuilder RegisterDatabase(IServiceContainer serviceContainer, IDictionary jobArgsDictionary, string connectionStringArgName) + { + if (serviceContainer == null) + { + throw new ArgumentNullException(nameof(serviceContainer)); + } + + if (jobArgsDictionary == null) + { + throw new ArgumentNullException(nameof(jobArgsDictionary)); + } + + if (string.IsNullOrEmpty(connectionStringArgName)) + { + throw new ArgumentException("Argument cannot be null or empty.", nameof(connectionStringArgName)); + } + + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var connectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, connectionStringArgName); + + return RegisterDatabase(connectionStringArgName, connectionString, secretInjector); + } + + /// + /// Register a job database at initialization time. Each call should overwrite any existing + /// registration because calls on every iteration. + /// + /// ConnectionStringBuilder, used for diagnostics. + private SqlConnectionStringBuilder RegisterDatabase(string name, string connectionString, ISecretInjector secretInjector) + { + var connectionFactory = new AzureSqlConnectionFactory(connectionString, secretInjector, Logger); + _sqlConnectionFactories[name] = connectionFactory; + + Task.Run(() => TestConnection(connectionFactory)).Wait(); + + return connectionFactory.SqlConnectionStringBuilder; + } + + /// + /// Create a SqlConnection, for use by validation jobs. + /// + public Task CreateSqlConnectionAsync() + where T : IDbConfiguration + { + var name = nameof(T); + if (!_sqlConnectionFactories.ContainsKey(name)) + { + throw new InvalidOperationException($"Database {name} has not been registered."); + } + + return _sqlConnectionFactories[name].CreateAsync(); + } + + /// + /// Synchronous creation of a SqlConnection, for use by validation jobs. + /// + public SqlConnection CreateSqlConnection() + where T : IDbConfiguration + { + return Task.Run(() => CreateSqlConnectionAsync()).Result; + } + + /// + /// Creates and opens a SqlConnection, for use by non-validation jobs. + /// + public Task OpenSqlConnectionAsync(string connectionStringArgName) + { + if (string.IsNullOrEmpty(connectionStringArgName)) + { + throw new ArgumentException("Argument cannot be null or empty.", nameof(connectionStringArgName)); + } + + if (!_sqlConnectionFactories.ContainsKey(connectionStringArgName)) + { + throw new InvalidOperationException($"Database {connectionStringArgName} has not been registered."); + } + + return _sqlConnectionFactories[connectionStringArgName].OpenAsync(); + } + public abstract void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary); public abstract Task Run(); diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj index 4e1ca11ca..1c1527895 100644 --- a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj +++ b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj @@ -49,7 +49,6 @@ - @@ -74,13 +73,16 @@ 1.50.2 - 2.25.0 + 2.27.0 - 2.26.0-master-33196 + 2.27.0 - 2.25.0 + 2.27.0 + + + 2.27.0 4.3.3 diff --git a/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs b/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs index e782f79de..7e47c419c 100644 --- a/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs +++ b/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs @@ -7,6 +7,19 @@ namespace NuGet.Services.Revalidate { public class RevalidationConfiguration { + /// + /// The lower limit for the desired package event rate (includes package pushes, lists, unlists, and revalidations). + /// If the ingestion pipeline remains healthy, the job will increase its rate over time. If the ingestion pipeline becomes + /// unhealthy, the job will reset its rate to this value. + /// + public int MinPackageEventRate { get; set; } + + /// + /// The revalidation job will speed up over time. This is the upper limit for the desired package event + /// rate (includes package pushes, lists, unlists, and revalidations). + /// + public int MaxPackageEventRate { get; set; } + /// /// The time before the revalidation job restarts itself. /// diff --git a/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs b/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs index b4cd2ad75..0a818cb88 100644 --- a/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs +++ b/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs @@ -15,20 +15,21 @@ public class InitializationManager { private static int BatchSize = 1000; - private readonly IRevalidationStateService _revalidationState; + private readonly IRevalidationJobStateService _jobState; + private readonly IPackageRevalidationStateService _packageState; private readonly IPackageFinder _packageFinder; private readonly InitializationConfiguration _config; private readonly ILogger _logger; public InitializationManager( - IRevalidationStateService revalidationState, + IRevalidationJobStateService jobState, + IPackageRevalidationStateService packageState, IPackageFinder packageFinder, InitializationConfiguration config, ILogger logger) { - // TODO: Accept service for settings (IsInitialized, etc...) - // See: https://github.com/NuGet/Engineering/issues/1440 - _revalidationState = revalidationState ?? throw new ArgumentNullException(nameof(revalidationState)); + _jobState = jobState ?? throw new ArgumentNullException(nameof(jobState)); + _packageState = packageState ?? throw new ArgumentNullException(nameof(packageState)); _packageFinder = packageFinder ?? throw new ArgumentNullException(nameof(packageFinder)); _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -36,8 +37,13 @@ public InitializationManager( public async Task InitializeAsync() { - // TODO: Check "IsInitialized" setting. If true, error! - // See: https://github.com/NuGet/Engineering/issues/1440 + if (await _jobState.IsInitializedAsync()) + { + _logger.LogError("Attempted to initialize the revalidation job when it is already initialized!"); + + throw new InvalidOperationException("Attempted to initialize the revalidation job when it is already initialized!"); + } + await ClearPackageRevalidationStateAsync(); // Find packages owned by Microsoft or preinstalled by Visual Studio. @@ -66,16 +72,20 @@ public async Task InitializeAsync() await InitializePackageSetAsync(PackageFinder.DependencySetName, dependencyPackages); await InitializePackageSetAsync(PackageFinder.RemainingSetName, remainingPackages); - // TODO: Set "IsInitialized" setting to true - // See: https://github.com/NuGet/Engineering/issues/1440 + await _jobState.MarkAsInitializedAsync(); } public async Task VerifyInitializationAsync() { - // TODO: Check "IsInitialized" setting. If false, error! - // See: https://github.com/NuGet/Engineering/issues/1440 + if (!await _jobState.IsInitializedAsync()) + { + _logger.LogError("Expected revalidation state to be initialized"); + + throw new Exception("Expected revalidation state to be initialized"); + } + var expectedCount = _packageFinder.AppropriatePackageCount(); - var actualCount = await _revalidationState.PackageRevalidationCountAsync(); + var actualCount = await _packageState.PackageRevalidationCountAsync(); if (actualCount != expectedCount) { @@ -93,7 +103,7 @@ private async Task ClearPackageRevalidationStateAsync() do { - removedRevalidations = await _revalidationState.RemoveRevalidationsAsync(BatchSize); + removedRevalidations = await _packageState.RemovePackageRevalidationsAsync(BatchSize); if (removedRevalidations > 0) { @@ -120,9 +130,19 @@ private async Task InitializePackageSetAsync(string setName, HashSet packag for (var chunkIndex = 0; chunkIndex < chunks.Count; chunkIndex++) { - // TODO: Check the kill switch - // See https://github.com/NuGet/Engineering/issues/1440 - _logger.LogInformation("Initializing chunk {Chunk} of {Chunks} for package set {SetName}...", + while (await _jobState.IsKillswitchActiveAsync()) + { + _logger.LogInformation( + "Delaying initialization of chunk {Chunk} of {Chunks} for package set {SetName} due to active killswitch", + chunkIndex + 1, + chunks.Count, + setName); + + await Task.Delay(_config.SleepDurationBetweenBatches); + } + + _logger.LogInformation( + "Initializing chunk {Chunk} of {Chunks} for package set {SetName}...", chunkIndex + 1, chunks.Count, setName); @@ -132,7 +152,8 @@ private async Task InitializePackageSetAsync(string setName, HashSet packag await InitializeRevalidationsAsync(chunk, versions); - _logger.LogInformation("Initialized chunk {Chunk} of {Chunks} for package set {SetName}", + _logger.LogInformation( + "Initialized chunk {Chunk} of {Chunks} for package set {SetName}", chunkIndex + 1, chunks.Count, setName); @@ -184,7 +205,7 @@ private async Task InitializeRevalidationsAsync( } } - await _revalidationState.AddPackageRevalidationsAsync(revalidations); + await _packageState.AddPackageRevalidationsAsync(revalidations); } } } diff --git a/src/NuGet.Services.Revalidate/Job.cs b/src/NuGet.Services.Revalidate/Job.cs index 2590ba358..5efb0e7bc 100644 --- a/src/NuGet.Services.Revalidate/Job.cs +++ b/src/NuGet.Services.Revalidate/Job.cs @@ -109,7 +109,9 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddTransient(); services.AddTransient(); - services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); // Initialization services.AddTransient(); diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj index 20c23ccfc..56c1b21f8 100644 --- a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj +++ b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj @@ -54,14 +54,17 @@ - + + + - + + @@ -99,6 +102,11 @@ Validation.PackageSigning.Core + + + 2.27.0 + + ..\..\build diff --git a/src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs b/src/NuGet.Services.Revalidate/Services/IPackageRevalidationStateService.cs similarity index 73% rename from src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs rename to src/NuGet.Services.Revalidate/Services/IPackageRevalidationStateService.cs index c0180cff6..f72e09ecc 100644 --- a/src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs +++ b/src/NuGet.Services.Revalidate/Services/IPackageRevalidationStateService.cs @@ -7,14 +7,8 @@ namespace NuGet.Services.Revalidate { - public interface IRevalidationStateService + public interface IPackageRevalidationStateService { - /// - /// Check whether the killswitch has been activated. If it has, all revalidation operations should be halted. - /// - /// Whether the killswitch has been activated. - Task IsKillswitchActiveAsync(); - /// /// Add the new revalidations to the database. /// @@ -22,10 +16,10 @@ public interface IRevalidationStateService Task AddPackageRevalidationsAsync(IReadOnlyList revalidations); /// - /// Remove revalidations from the database. + /// Remove package revalidation entities from the database. /// /// A task that returns the number of revalidations that have been removed. - Task RemoveRevalidationsAsync(int max); + Task RemovePackageRevalidationsAsync(int max); /// /// Count the number of package revalidations in the database. @@ -33,11 +27,17 @@ public interface IRevalidationStateService /// The count of package revalidations in the database. Task PackageRevalidationCountAsync(); + /// + /// Count the number of package revalidations that were enqueued in the past hour. + /// + /// The number of enqueued revalidations. + Task CountRevalidationsEnqueuedInPastHourAsync(); + /// /// Update the package revalidation and mark is as enqueued. /// /// The revalidation to update. /// A task that completes once the revalidation has been updated. - Task MarkRevalidationAsEnqueuedAsync(PackageRevalidation revalidation); + Task MarkPackageRevalidationAsEnqueuedAsync(PackageRevalidation revalidation); } } diff --git a/src/NuGet.Services.Revalidate/Services/IRevalidationJobStateService.cs b/src/NuGet.Services.Revalidate/Services/IRevalidationJobStateService.cs new file mode 100644 index 000000000..0723cee0e --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/IRevalidationJobStateService.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Revalidate +{ + /// + /// The state shared between the Gallery and the revalidation job. + /// + public interface IRevalidationJobStateService + { + /// + /// Check whether the revalidation state has been initialized. + /// + /// Whether the revalidation state has been initialized. + Task IsInitializedAsync(); + + /// + /// Update the settings to mark the revalidation job as initialized. + /// + /// A task that completes once the settings have been updated. + Task MarkAsInitializedAsync(); + + /// + /// Check whether the killswitch has been activated. If it has, all revalidation operations should be halted. + /// + /// Whether the killswitch has been activated. + Task IsKillswitchActiveAsync(); + + /// + /// Determine the desired package event rate per hour. Package events include package pushes, + /// edits, and revalidations. + /// + /// The desired maximum number of package events per hour. + Task GetDesiredPackageEventRateAsync(); + + /// + /// Reset the desired package event rate to the configured minimum. + /// + /// A task that completes once the rate has been reset. + Task ResetDesiredPackageEventRateAsync(); + + /// + /// Increment the desired package event rate per hour. + /// + /// A task that completes once the rate has been incremented. + Task IncreaseDesiredPackageEventRateAsync(); + } +} diff --git a/src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs b/src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs index 1d03969e6..e1870321f 100644 --- a/src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs +++ b/src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs @@ -13,19 +13,6 @@ public interface IRevalidationThrottler /// If true, no more revalidations should be performed. Task IsThrottledAsync(); - /// - /// Reset the capacity to the configured minimum value. Call this when the service's status is degraded to - /// throttle the revalidations. - /// - /// A task that completes once the capacity theshold has been reset. - Task ResetCapacityAsync(); - - /// - /// Increase the revalidation capacity by one revalidation per minute. - /// - /// A task taht completes once the capacity has been increased. - Task IncreaseCapacityAsync(); - /// /// Delay the current task to achieve the desired revalidation rate. /// diff --git a/src/NuGet.Services.Revalidate/Services/ITelemetryService.cs b/src/NuGet.Services.Revalidate/Services/ITelemetryService.cs index 01c487039..881578911 100644 --- a/src/NuGet.Services.Revalidate/Services/ITelemetryService.cs +++ b/src/NuGet.Services.Revalidate/Services/ITelemetryService.cs @@ -1,13 +1,13 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; +using NuGet.Services.Logging; namespace NuGet.Services.Revalidate { public interface ITelemetryService { - IDisposable TrackDurationToStartNextRevalidation(); + DurationMetric TrackStartNextRevalidationOperation(); void TrackPackageRevalidationMarkedAsCompleted(string packageId, string normalizedVersion); diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationStateService.cs b/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs similarity index 78% rename from src/NuGet.Services.Revalidate/Services/RevalidationStateService.cs rename to src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs index 1220486a1..8f7a79c1a 100644 --- a/src/NuGet.Services.Revalidate/Services/RevalidationStateService.cs +++ b/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs @@ -12,23 +12,17 @@ namespace NuGet.Services.Revalidate { - public class RevalidationStateService : IRevalidationStateService + public class PackageRevalidationStateService : IPackageRevalidationStateService { private readonly IValidationEntitiesContext _context; - private readonly ILogger _logger; + private readonly ILogger _logger; - public RevalidationStateService(IValidationEntitiesContext context, ILogger logger) + public PackageRevalidationStateService(IValidationEntitiesContext context, ILogger logger) { _context = context ?? throw new ArgumentNullException(nameof(context)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public Task IsKillswitchActiveAsync() - { - // TODO - return Task.FromResult(false); - } - public async Task AddPackageRevalidationsAsync(IReadOnlyList revalidations) { var validationContext = _context as ValidationEntitiesContext; @@ -53,7 +47,7 @@ public async Task AddPackageRevalidationsAsync(IReadOnlyList RemoveRevalidationsAsync(int max) + public async Task RemovePackageRevalidationsAsync(int max) { var revalidations = await _context.PackageRevalidations .Take(max) @@ -77,7 +71,16 @@ public async Task PackageRevalidationCountAsync() return await _context.PackageRevalidations.CountAsync(); } - public async Task MarkRevalidationAsEnqueuedAsync(PackageRevalidation revalidation) + public async Task CountRevalidationsEnqueuedInPastHourAsync() + { + var lowerBound = DateTime.UtcNow.Subtract(TimeSpan.FromHours(1)); + + return await _context.PackageRevalidations + .Where(r => r.Enqueued >= lowerBound) + .CountAsync(); + } + + public async Task MarkPackageRevalidationAsEnqueuedAsync(PackageRevalidation revalidation) { try { diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs b/src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs new file mode 100644 index 000000000..2ace054da --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs @@ -0,0 +1,124 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGetGallery; + +namespace NuGet.Services.Revalidate +{ + public class RevalidationJobStateService : IRevalidationJobStateService + { + private readonly IRevalidationStateService _state; + private readonly RevalidationConfiguration _config; + private readonly ILogger _logger; + + public RevalidationJobStateService( + IRevalidationStateService state, + RevalidationConfiguration config, + ILogger logger) + { + _state = state ?? throw new ArgumentNullException(nameof(state)); + _config = config ?? throw new ArgumentNullException(nameof(config)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task IsInitializedAsync() + { + return await GetStateValue(s => s.IsInitialized); + } + + public async Task MarkAsInitializedAsync() + { + _logger.LogInformation("Updating state as initialized"); + + await _state.UpdateStateAsync(s => s.IsInitialized = true); + } + + public async Task IsKillswitchActiveAsync() + { + return await GetStateValue(s => s.IsKillswitchActive); + } + + public async Task GetDesiredPackageEventRateAsync() + { + // Ensure the desired rate is within the configured lower and upper bounds. A desired rate that's outside + // the bounds indicates that the job was redeployed with different configuration values. + var finalState = await _state.MaybeUpdateStateAsync(state => + { + if (state.DesiredPackageEventRate < _config.MinPackageEventRate) + { + _logger.LogInformation( + "Overriding desired package event rate {ToRate} from {FromRate}", + _config.MinPackageEventRate, + state.DesiredPackageEventRate); + + state.DesiredPackageEventRate = _config.MinPackageEventRate; + + return true; + } + else if (state.DesiredPackageEventRate > _config.MaxPackageEventRate) + { + _logger.LogInformation( + "Overriding desired package event rate {ToRate} from {FromRate}", + _config.MaxPackageEventRate, + state.DesiredPackageEventRate); + + state.DesiredPackageEventRate = _config.MaxPackageEventRate; + + return true; + } + + // The rate is within the expected bounds. Don't update anything. + return false; + }); + + return finalState.DesiredPackageEventRate; + } + + public async Task ResetDesiredPackageEventRateAsync() + { + await _state.UpdateStateAsync(state => + { + _logger.LogInformation( + "Resetting desired package event rate to {ToRate} from {FromRate}", + _config.MinPackageEventRate, + state.DesiredPackageEventRate); + + state.DesiredPackageEventRate = _config.MinPackageEventRate; + }); + } + + public async Task IncreaseDesiredPackageEventRateAsync() + { + await _state.MaybeUpdateStateAsync(state => + { + // Don't update the state if we've reached the upper limit. + if (state.DesiredPackageEventRate == _config.MaxPackageEventRate) + { + _logger.LogInformation( + "Desired package event rate is at configured maximum of {MaxRate} per hour", + _config.MaxPackageEventRate); + + return false; + } + + var nextRate = Math.Min(_config.MaxPackageEventRate, state.DesiredPackageEventRate + 1); + + _logger.LogInformation( + "Increasing desired package event rate to {ToRate} from {FromRate}", + nextRate, + state.DesiredPackageEventRate); + + state.DesiredPackageEventRate = nextRate; + return true; + }); + } + + private async Task GetStateValue(Func callback) + { + return callback(await _state.GetStateAsync()); + } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationOperation.cs b/src/NuGet.Services.Revalidate/Services/RevalidationOperation.cs new file mode 100644 index 000000000..9a85e08cc --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/RevalidationOperation.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Revalidate +{ + public class StartNextRevalidationOperation + { + /// + /// The result of attempting to start the next revalidation. + /// + public RevalidationResult Result { get; set; } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationService.cs b/src/NuGet.Services.Revalidate/Services/RevalidationService.cs index 46db515de..b196bf3d0 100644 --- a/src/NuGet.Services.Revalidate/Services/RevalidationService.cs +++ b/src/NuGet.Services.Revalidate/Services/RevalidationService.cs @@ -11,7 +11,8 @@ namespace NuGet.Services.Revalidate { public class RevalidationService : IRevalidationService { - private readonly IRevalidationStateService _state; + private readonly IRevalidationJobStateService _jobState; + private readonly IPackageRevalidationStateService _packageState; private readonly ISingletonService _singletonService; private readonly IRevalidationThrottler _throttler; private readonly IHealthService _healthService; @@ -22,7 +23,8 @@ public class RevalidationService : IRevalidationService private readonly ILogger _logger; public RevalidationService( - IRevalidationStateService state, + IRevalidationJobStateService jobState, + IPackageRevalidationStateService packageState, ISingletonService singletonService, IRevalidationThrottler throttler, IHealthService healthService, @@ -32,7 +34,8 @@ public RevalidationService( ITelemetryService telemetryService, ILogger logger) { - _state = state ?? throw new ArgumentNullException(nameof(state)); + _jobState = jobState ?? throw new ArgumentNullException(nameof(jobState)); + _packageState = packageState ?? throw new ArgumentNullException(nameof(packageState)); _singletonService = singletonService ?? throw new ArgumentNullException(nameof(singletonService)); _throttler = throttler ?? throw new ArgumentNullException(nameof(throttler)); _healthService = healthService ?? throw new ArgumentNullException(nameof(healthService)); @@ -45,6 +48,13 @@ public RevalidationService( public async Task RunAsync() { + if (!await _jobState.IsInitializedAsync()) + { + _logger.LogError("The revalidation service must be initialized before running revalidations"); + + throw new InvalidOperationException("The revalidation service must be initialized before running revalidations"); + } + var runTime = Stopwatch.StartNew(); do @@ -83,7 +93,19 @@ public async Task RunAsync() public async Task StartNextRevalidationAsync() { - using (_telemetryService.TrackDurationToStartNextRevalidation()) + using (var operation = _telemetryService.TrackStartNextRevalidationOperation()) + { + var result = await StartNextRevalidationInternalAsync(); + + operation.Properties.Result = result; + + return result; + } + } + + public async Task StartNextRevalidationInternalAsync() + { + try { // Don't start a revalidation if the job has been deactivated, if the ingestion pipeline is unhealthy, // or if we have reached our quota of desired revalidations. @@ -98,7 +120,7 @@ public async Task StartNextRevalidationAsync() } // Everything is in tip-top shape! Increase the throttling quota and start the next revalidation. - await _throttler.IncreaseCapacityAsync(); + await _jobState.IncreaseDesiredPackageEventRateAsync(); var revalidation = await _revalidationQueue.NextOrNullAsync(); if (revalidation == null) @@ -110,6 +132,12 @@ public async Task StartNextRevalidationAsync() return await StartRevalidationAsync(revalidation); } + catch (Exception e) + { + _logger.LogError(0, e, "Failed to start next validation due to exception, retry later..."); + + return RevalidationResult.RetryLater; + } } private async Task CanStartRevalidationAsync() @@ -121,7 +149,7 @@ public async Task StartNextRevalidationAsync() return RevalidationResult.UnrecoverableError; } - if (await _state.IsKillswitchActiveAsync()) + if (await _jobState.IsKillswitchActiveAsync()) { _logger.LogWarning("Revalidation killswitch has been activated, retry later..."); @@ -137,14 +165,14 @@ public async Task StartNextRevalidationAsync() if (!await _healthService.IsHealthyAsync()) { - _logger.LogWarning("Service appears to be unhealthy, resetting throttling capacity and retry later..."); + _logger.LogWarning("Service appears to be unhealthy, resetting the desired package event rate. Retry later..."); - await _throttler.ResetCapacityAsync(); + await _jobState.ResetDesiredPackageEventRateAsync(); return RevalidationResult.RetryLater; } - if (await _state.IsKillswitchActiveAsync()) + if (await _jobState.IsKillswitchActiveAsync()) { _logger.LogWarning("Revalidation killswitch has been activated after the throttle and health check, retry later..."); @@ -162,7 +190,7 @@ private async Task StartRevalidationAsync(PackageRevalidatio revalidation.ValidationTrackingId.Value); await _validationEnqueuer.StartValidationAsync(message); - await _state.MarkRevalidationAsEnqueuedAsync(revalidation); + await _packageState.MarkPackageRevalidationAsEnqueuedAsync(revalidation); _telemetryService.TrackPackageRevalidationStarted(revalidation.PackageId, revalidation.PackageNormalizedVersion); diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs b/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs index b736d5863..4041832f7 100644 --- a/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs +++ b/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs @@ -9,41 +9,42 @@ namespace NuGet.Services.Revalidate { public class RevalidationThrottler : IRevalidationThrottler { + private readonly IRevalidationJobStateService _jobState; + private readonly IPackageRevalidationStateService _packageState; private readonly RevalidationConfiguration _config; private readonly ILogger _logger; - public RevalidationThrottler(RevalidationConfiguration config, ILogger logger) + public RevalidationThrottler( + IRevalidationJobStateService jobState, + IPackageRevalidationStateService packageState, + RevalidationConfiguration config, + ILogger logger) { + _jobState = jobState ?? throw new ArgumentNullException(nameof(jobState)); + _packageState = packageState ?? throw new ArgumentNullException(nameof(packageState)); _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public Task IsThrottledAsync() + public async Task IsThrottledAsync() { - // TODO: - // Calculate desired event rate - // Calculate current event rate (# of revalidations + Gallery actions) - // Compare desired event rate to configured event rate. If configured rate is higher, update desired event rate. - // If current event rate is greater than or equal to desired event rate, return true; - return Task.FromResult(false); - } + var desiredRate = await _jobState.GetDesiredPackageEventRateAsync(); + var recentGalleryEvents = await CountGalleryEventsInPastHourAsync(); + var recentRevalidations = await _packageState.CountRevalidationsEnqueuedInPastHourAsync(); - public Task ResetCapacityAsync() - { - return Task.CompletedTask; - } + var revalidationQuota = desiredRate - recentRevalidations - recentGalleryEvents; - public Task IncreaseCapacityAsync() - { - return Task.CompletedTask; + return (revalidationQuota <= 0); } public async Task DelayUntilNextRevalidationAsync() { - // TODO: Calculate sleep duration to achieve desired event rate. - _logger.LogInformation("Delaying until next revalidation by sleeping for 5 minutes..."); + var desiredHourlyRate = await _jobState.GetDesiredPackageEventRateAsync(); + var sleepDuration = TimeSpan.FromHours(1.0 / desiredHourlyRate); - await Task.Delay(TimeSpan.FromMinutes(5)); + _logger.LogInformation("Delaying until next revalidation by sleeping for {SleepDuration}...", sleepDuration); + + await Task.Delay(sleepDuration); } public async Task DelayUntilRevalidationRetryAsync() @@ -54,5 +55,22 @@ public async Task DelayUntilRevalidationRetryAsync() await Task.Delay(_config.RetryLaterSleep); } + + private Task CountGalleryEventsInPastHourAsync() + { + // TODO: Count the number of package pushes, lists, and unlists. + // Run this AI query: + // + // customMetrics | where name == "PackagePush" or name == "PackageUnlisted" or name == "PackageListed" | summarize sum(value) + // + // Using this HTTP request: + // + // GET /v1/apps/46f13c7d-635f-42c3-8120-593edeaad426/query?timespan=P1D&query=customMetrics%20%7C%20where%20name%20%3D%3D%20%22PackagePush%22%20or%20name%20%3D%3D%20%22PackageUnlisted%22%20or%20name%20%3D%3D%20%22PackageListed%22%20%7C%20summarize%20sum(value)%20 HTTP/1.1 + // Host: api.applicationinsights.io + // x-api-key: my-super-secret-api-key + // + // See: https://dev.applicationinsights.io/quickstart + return Task.FromResult(0); + } } } diff --git a/src/NuGet.Services.Revalidate/Services/TelemetryService.cs b/src/NuGet.Services.Revalidate/Services/TelemetryService.cs index 9ce792992..1ea15a0e6 100644 --- a/src/NuGet.Services.Revalidate/Services/TelemetryService.cs +++ b/src/NuGet.Services.Revalidate/Services/TelemetryService.cs @@ -11,12 +11,13 @@ public class TelemetryService : ITelemetryService { private const string RevalidationPrefix = "Revalidation."; - private const string DurationToStartNextRevalidation = RevalidationPrefix + "DurationToStartNextRevalidation"; + private const string DurationToStartNextRevalidation = RevalidationPrefix + "DurationToStartNextRevalidationSeconds"; private const string PackageRevalidationMarkedAsCompleted = RevalidationPrefix + "PackageRevalidationMarkedAsCompleted"; private const string PackageRevalidationStarted = RevalidationPrefix + "PackageRevalidationStarted"; private const string PackageId = "PackageId"; private const string NormalizedVersion = "NormalizedVersion"; + private const string Result = "Result"; private readonly ITelemetryClient _client; @@ -25,9 +26,15 @@ public TelemetryService(ITelemetryClient client) _client = client ?? throw new ArgumentNullException(nameof(client)); } - public IDisposable TrackDurationToStartNextRevalidation() + public DurationMetric TrackStartNextRevalidationOperation() { - return _client.TrackDuration(DurationToStartNextRevalidation); + return _client.TrackDuration( + DurationToStartNextRevalidation, + new StartNextRevalidationOperation(), + o => new Dictionary + { + { Result, o.Result.ToString() } + }); } public void TrackPackageRevalidationMarkedAsCompleted(string packageId, string normalizedVersion) diff --git a/src/NuGet.Services.Revalidate/Settings/dev.json b/src/NuGet.Services.Revalidate/Settings/dev.json index 2964de561..d79310f37 100644 --- a/src/NuGet.Services.Revalidate/Settings/dev.json +++ b/src/NuGet.Services.Revalidate/Settings/dev.json @@ -9,18 +9,30 @@ "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:01" }, + + "MinPackageEventRate": 120, + "MaxPackageEventRate": 500, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" } }, + "Storage": { + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "Container": "revalidations" + }, + "GalleryDb": { "ConnectionString": "Data Source=tcp:#{Jobs.validation.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Integrated Security=False;User ID=$$Dev-GalleryDBReadOnly-UserName$$;Password=$$Dev-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True" }, "ValidationDb": { "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-dev-validation;Integrated Security=False;User ID=$$Dev-ValidationDBWriter-UserName$$;Password=$$Dev-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, + "ValidationStorage": { + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$" + }, "ServiceBus": { "ConnectionString": "Endpoint=sb://nugetdev.servicebus.windows.net/;SharedAccessKeyName=gallery;SharedAccessKey=$$Dev-ServiceBus-SharedAccessKey-Validation-GallerySender$$", "TopicPath": "validation" diff --git a/src/NuGet.Services.Revalidate/Settings/int.json b/src/NuGet.Services.Revalidate/Settings/int.json index 25c5abb5c..acebe3e27 100644 --- a/src/NuGet.Services.Revalidate/Settings/int.json +++ b/src/NuGet.Services.Revalidate/Settings/int.json @@ -9,6 +9,10 @@ "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:30" }, + + "MinPackageEventRate": 120, + "MaxPackageEventRate": 500, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" @@ -21,6 +25,9 @@ "ValidationDb": { "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-int-validation;Integrated Security=False;User ID=$$Int-ValidationDBWriter-UserName$$;Password=$$Int-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, + "ValidationStorage": { + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$" + }, "ServiceBus": { "ConnectionString": "Endpoint=sb://nugetint.servicebus.windows.net/;SharedAccessKeyName=gallery;SharedAccessKey=$$Int-ServiceBus-SharedAccessKey-Validation-GallerySender$$", "TopicPath": "validation" diff --git a/src/NuGet.Services.Revalidate/Settings/prod.json b/src/NuGet.Services.Revalidate/Settings/prod.json index 519b9d11f..f873a636d 100644 --- a/src/NuGet.Services.Revalidate/Settings/prod.json +++ b/src/NuGet.Services.Revalidate/Settings/prod.json @@ -9,6 +9,10 @@ "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:30" }, + + "MinPackageEventRate": 120, + "MaxPackageEventRate": 500, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" @@ -21,6 +25,9 @@ "ValidationDb": { "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-prod-validation;Integrated Security=False;User ID=$$Prod-ValidationDBWriter-UserName$$;Password=$$Prod-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, + "ValidationStorage": { + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$" + }, "ServiceBus": { "ConnectionString": "Endpoint=sb://nugetprod.servicebus.windows.net/;SharedAccessKeyName=gallery;SharedAccessKey=$$Prod-ServiceBus-SharedAccessKey-Validation-GallerySender$$", "TopicPath": "validation" diff --git a/src/NuGet.Services.Revalidate/readme.md b/src/NuGet.Services.Revalidate/readme.md new file mode 100644 index 000000000..c9a28c051 --- /dev/null +++ b/src/NuGet.Services.Revalidate/readme.md @@ -0,0 +1,41 @@ +# The Revalidate Job + +This job enqueues packages revalidation as fast as possible without affecting the +health of NuGet's ingestion pipeline. It does so in two phases: + +1. Initialization phase - the job determines which packages should be revalidated. +2. Revalidation phase - packages are enqueued for revalidations + +The initialization phase MUST complete before the revalidation phase is started. + +# The Initialization Phase + +To initialize the job, run: + +``` +NuGet.Services.Revalidate.exe ^ + -Configuration "C:\Path\to\config.json" ^ + -Initialize + -VerifyInitialization + -Once +``` + +This will figure which packages should be revalidated, and the order that packages +should be revalidated. Packages are prioritized by: + +1. Packages owned by the `Microsoft` account +2. Packages installed by Visual Studio or by the .NET Core SDK +3. All remaining packages + +Pending package revalidations are stored in the `PackageRevalidations` +table, in order of priority. + +# The Revalidation Phase + +To enqueue revalidations, run: + +``` +NuGet.Services.Revalidate.exe ^ + -Configuration "C:\Path\to\config.json" ^ + -Initialize +``` \ No newline at end of file diff --git a/src/NuGet.Services.Validation.Orchestrator/Job.cs b/src/NuGet.Services.Validation.Orchestrator/Job.cs index 8e6c9f831..b3c5638c3 100644 --- a/src/NuGet.Services.Validation.Orchestrator/Job.cs +++ b/src/NuGet.Services.Validation.Orchestrator/Job.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.Common; using System.Net; using System.Net.Http; using System.Reflection; @@ -31,7 +30,6 @@ using NuGet.Services.KeyVault; using NuGet.Services.Logging; using NuGet.Services.ServiceBus; -using NuGet.Services.Sql; using NuGet.Services.Validation.Orchestrator.PackageSigning.ScanAndSign; using NuGet.Services.Validation.Orchestrator.Telemetry; using NuGet.Services.Validation.PackageSigning.ProcessSignature; @@ -86,6 +84,12 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(_serviceProvider); + RegisterDatabase(_serviceProvider); + } ConfigurationValidated = false; } @@ -158,15 +162,6 @@ private void ConfigureLibraries(IServiceCollection services) services.AddLogging(); } - private DbConnection CreateDbConnection(IServiceProvider serviceProvider) where T : IDbConfiguration - { - var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; - var connectionFactory = new AzureSqlConnectionFactory(connectionString, - serviceProvider.GetRequiredService()); - - return Task.Run(() => connectionFactory.CreateAsync()).Result; - } - private void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); @@ -184,15 +179,15 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo services.AddTransient(); services.AddTransient(); - + services.AddScoped(serviceProvider => new NuGetGallery.EntitiesContext( - CreateDbConnection(serviceProvider), + CreateSqlConnection(), readOnly: false) ); services.AddScoped(serviceProvider => new ValidationEntitiesContext( - CreateDbConnection(serviceProvider))); + CreateSqlConnection())); services.AddScoped(serviceProvider => serviceProvider.GetRequiredService()); diff --git a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj index ebf1fb92c..e7ba23387 100644 --- a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj +++ b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj @@ -123,9 +123,6 @@ 1.2.0 - - 2.26.0-master-34394 - 2.26.0-master-34394 diff --git a/src/NuGet.SupportRequests.Notifications/Job.cs b/src/NuGet.SupportRequests.Notifications/Job.cs index 99de91ce9..8f3292405 100644 --- a/src/NuGet.SupportRequests.Notifications/Job.cs +++ b/src/NuGet.SupportRequests.Notifications/Job.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Threading.Tasks; using NuGet.Jobs; @@ -24,11 +25,19 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenSupportSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.SourceDatabase); } public override async Task Run() { - var scheduledTask = ScheduledTaskFactory.Create(_serviceContainer, _jobArgsDictionary, LoggerFactory); + var scheduledTask = ScheduledTaskFactory.Create(_serviceContainer, _jobArgsDictionary, + OpenSupportSqlConnectionAsync, LoggerFactory); await scheduledTask.RunAsync(); } diff --git a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj index a72a38207..44a43935d 100644 --- a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj +++ b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj @@ -106,9 +106,6 @@ 9.0.1 - - 2.25.0-master-30453 - diff --git a/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs b/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs index 67586efb9..42834faf2 100644 --- a/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs +++ b/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs @@ -4,6 +4,8 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; namespace NuGet.SupportRequests.Notifications @@ -12,7 +14,11 @@ internal class ScheduledTaskFactory { private const string _tasksNamespace = "NuGet.SupportRequests.Notifications.Tasks"; - public static IScheduledTask Create(IServiceContainer serviceContainer, IDictionary jobArgsDictionary, ILoggerFactory loggerFactory) + public static IScheduledTask Create( + IServiceContainer serviceContainer, + IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, + ILoggerFactory loggerFactory) { if (jobArgsDictionary == null) { @@ -25,7 +31,8 @@ public static IScheduledTask Create(IServiceContainer serviceContainer, IDiction } var scheduledTaskName = jobArgsDictionary[JobArgumentNames.ScheduledTask]; - var scheduledTask = GetTaskOfType(scheduledTaskName, serviceContainer, jobArgsDictionary, loggerFactory); + var scheduledTask = GetTaskOfType(scheduledTaskName, serviceContainer, jobArgsDictionary, + openSupportSqlConnectionAsync, loggerFactory); return scheduledTask; } @@ -34,6 +41,7 @@ private static IScheduledTask GetTaskOfType( string taskName, IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) { if (string.IsNullOrEmpty(taskName)) @@ -51,7 +59,7 @@ private static IScheduledTask GetTaskOfType( IScheduledTask scheduledTask; if (scheduledTaskType != null && typeof(IScheduledTask).IsAssignableFrom(scheduledTaskType)) { - var args = new object[] { serviceContainer, jobArgsDictionary, loggerFactory }; + var args = new object[] { serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory }; scheduledTask = (IScheduledTask)Activator.CreateInstance(scheduledTaskType, args); } else diff --git a/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs b/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs index 1dcea245b..d4c3f8f20 100644 --- a/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs +++ b/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; using NuGet.SupportRequests.Notifications.Models; namespace NuGet.SupportRequests.Notifications @@ -20,29 +19,24 @@ internal class SupportRequestRepository private const string _parameterNamePagerDutyUsername = "pagerDutyUserName"; private readonly DateTime _defaultSqlDateTime = new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc); private readonly ILogger _logger; - private readonly ISqlConnectionFactory _supportDbConnectionFactory; + private readonly Func> _openSupportSqlConnectionAsync; public SupportRequestRepository( - ILoggerFactory loggerFactory, - ISqlConnectionFactory supportDbConnectionFactory) + Func> openSupportSqlConnectionAsync, + ILoggerFactory loggerFactory) { if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } - if (supportDbConnectionFactory == null) - { - throw new ArgumentNullException(nameof(supportDbConnectionFactory)); - } - _logger = loggerFactory.CreateLogger(); - _supportDbConnectionFactory = supportDbConnectionFactory; + _openSupportSqlConnectionAsync = openSupportSqlConnectionAsync; } - internal async Task OpenConnectionAsync() + internal async Task OpenSupportSqlConnectionAsync() { - var connection = await _supportDbConnectionFactory.CreateAsync(); + var connection = await _openSupportSqlConnectionAsync(); connection.InfoMessage += OnSqlConnectionInfoMessage; return connection; @@ -181,7 +175,7 @@ private async Task EnsureConnectionOpenAsync(SqlConnection connec { if (connection == null) { - connection = await OpenConnectionAsync(); + connection = await OpenSupportSqlConnectionAsync(); } else if (connection.State != ConnectionState.Open) { diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs index 35892f72d..82d9b09a1 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -25,8 +26,9 @@ internal class OnCallDailyNotificationTask public OnCallDailyNotificationTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) - : base(serviceContainer, jobArgsDictionary, loggerFactory) + : base(serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory) { var pagerDutyConfiguration = new PagerDutyConfiguration( jobArgsDictionary[_argumentNamePagerDutyAccountName], @@ -44,7 +46,7 @@ protected override async Task BuildNotification( var targetEmailAddress = string.Format(_targetEmailAddressFormat, onCallAlias); List unresolvedIssues; - using (var connection = await supportRequestRepository.OpenConnectionAsync()) + using (var connection = await supportRequestRepository.OpenSupportSqlConnectionAsync()) { unresolvedIssues = await supportRequestRepository.GetUnresolvedIssues(connection, onCallAlias); } diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs index 6976ed0bb..51ffd7d5d 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs @@ -4,10 +4,9 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using NuGet.SupportRequests.Notifications.Notifications; using NuGet.SupportRequests.Notifications.Services; using NuGet.SupportRequests.Notifications.Templates; @@ -24,6 +23,7 @@ internal abstract class SupportRequestsNotificationScheduledTask protected SupportRequestsNotificationScheduledTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) { if (jobArgsDictionary == null) @@ -38,15 +38,12 @@ protected SupportRequestsNotificationScheduledTask( var smtpUri = jobArgsDictionary[JobArgumentNames.SmtpUri]; _messagingService = new MessagingService(loggerFactory, smtpUri); - - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var supportDbConnectionString = jobArgsDictionary[JobArgumentNames.SourceDatabase]; - var supportDbConnectionFactory = new AzureSqlConnectionFactory(supportDbConnectionString, secretInjector); - _supportRequestRepository = new SupportRequestRepository(loggerFactory, supportDbConnectionFactory); + _supportRequestRepository = new SupportRequestRepository(openSupportSqlConnectionAsync, loggerFactory); } protected abstract Task BuildNotification(SupportRequestRepository supportRequestRepository, DateTime referenceTime); + protected abstract string BuildNotificationBody(string template, TNotification notification); public async Task RunAsync() diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs index 71a0f9ccf..efb3c9518 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -24,8 +25,9 @@ internal class WeeklySummaryNotificationTask public WeeklySummaryNotificationTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) - : base(serviceContainer, jobArgsDictionary, loggerFactory) + : base(serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory) { _targetEmailAddress = jobArgsDictionary[_argumentNameTargetEmailAddress]; } @@ -42,7 +44,7 @@ protected override async Task BuildNotification( var startDateUtcLastWeek = referenceTime.AddDays(-7); var startDateUtcPriorWeek = referenceTime.AddDays(-14); - using (var connection = await supportRequestRepository.OpenConnectionAsync()) + using (var connection = await supportRequestRepository.OpenSupportSqlConnectionAsync()) { unresolvedIssues = await supportRequestRepository.GetUnresolvedIssues(connection); diff --git a/src/PackageLagMonitor/Monitoring.PackageLag.csproj b/src/PackageLagMonitor/Monitoring.PackageLag.csproj index a41d91d01..daa00bca3 100644 --- a/src/PackageLagMonitor/Monitoring.PackageLag.csproj +++ b/src/PackageLagMonitor/Monitoring.PackageLag.csproj @@ -111,13 +111,13 @@ 0.5.0-CI-20180510-012541 - 2.26.0 + 2.27.0 - 2.25.0 + 2.27.0 - 2.25.0 + 2.27.0 4.3.3 diff --git a/src/Search.GenerateAuxiliaryData/Job.cs b/src/Search.GenerateAuxiliaryData/Job.cs index 4d0971f80..84ed5b8b4 100644 --- a/src/Search.GenerateAuxiliaryData/Job.cs +++ b/src/Search.GenerateAuxiliaryData/Job.cs @@ -11,12 +11,10 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { - internal class Job + public class Job : JobBase { private const string DefaultContainerName = "ng-search-data"; @@ -45,13 +43,8 @@ internal class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - - var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - var packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); - - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - var statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); var statisticsStorageAccount = CloudStorageAccount.Parse( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount)); @@ -69,14 +62,42 @@ public override void Init(IServiceContainer serviceContainer, IDictionary { - new VerifiedPackagesExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptVerifiedPackages, OutputNameVerifiedPackages), - new NestedJArrayExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptCuratedFeed, OutputNameCuratedFeed, Col0CuratedFeed, Col1CuratedFeed), - new NestedJArrayExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptOwners, OutputNameOwners, Col0Owners, Col1Owners), - new RankingsExporter(LoggerFactory.CreateLogger(), statisticsDbConnectionFactory, _destContainer, ScriptRankingsTotal, OutputNameRankings), - new BlobStorageExporter(LoggerFactory.CreateLogger(), _statisticsContainer, StatisticsReportName, _destContainer, StatisticsReportName) + new VerifiedPackagesExporter( + OpenGallerySqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptVerifiedPackages, OutputNameVerifiedPackages), + + new NestedJArrayExporter( + OpenGallerySqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptCuratedFeed, OutputNameCuratedFeed, Col0CuratedFeed, Col1CuratedFeed), + + new NestedJArrayExporter( + OpenGallerySqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptOwners, OutputNameOwners, Col0Owners, Col1Owners), + + new RankingsExporter( + OpenStatisticsSqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptRankingsTotal, OutputNameRankings), + + new BlobStorageExporter( + LoggerFactory.CreateLogger(), + _statisticsContainer, StatisticsReportName, _destContainer, StatisticsReportName) }; } + public Task OpenGallerySqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase); + } + + public Task OpenStatisticsSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); + } + public override async Task Run() { var failedExporters = new List(); diff --git a/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs b/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs index 83824718a..ff4cd505c 100644 --- a/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs +++ b/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs @@ -6,10 +6,10 @@ using System.Data; using System.Data.SqlClient; using System.Linq; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -17,11 +17,17 @@ class NestedJArrayExporter : SqlExporter { public string Col0 { get; } + public string Col1 { get; } + public string SqlScript { get; } - public NestedJArrayExporter(ILogger logger, ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + public NestedJArrayExporter( + Func> openGallerySqlConnectionAsync, + ILogger logger, + CloudBlobContainer defaultDestinationContainer, + string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1) + : base(openGallerySqlConnectionAsync, logger, defaultDestinationContainer, defaultName) { Col0 = defaultCol0; Col1 = defaultCol1; diff --git a/src/Search.GenerateAuxiliaryData/RankingsExporter.cs b/src/Search.GenerateAuxiliaryData/RankingsExporter.cs index 520812ff7..14ddb8bee 100644 --- a/src/Search.GenerateAuxiliaryData/RankingsExporter.cs +++ b/src/Search.GenerateAuxiliaryData/RankingsExporter.cs @@ -4,10 +4,10 @@ using System; using System.Data; using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -20,12 +20,12 @@ public sealed class RankingsExporter : SqlExporter private readonly string _rankingsTotalScript; public RankingsExporter( + Func> openStatisticsSqlConnectionAsync, ILogger logger, - ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultRankingsScript, string defaultName) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + : base(openStatisticsSqlConnectionAsync, logger, defaultDestinationContainer, defaultName) { _rankingsTotalScript = defaultRankingsScript; } diff --git a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj index e9d6dddac..d0a100c02 100644 --- a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj +++ b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj @@ -92,9 +92,6 @@ 9.0.1 - - 2.25.0-master-30453 - 7.1.2 diff --git a/src/Search.GenerateAuxiliaryData/SqlExporter.cs b/src/Search.GenerateAuxiliaryData/SqlExporter.cs index fedaa8227..652b1df4f 100644 --- a/src/Search.GenerateAuxiliaryData/SqlExporter.cs +++ b/src/Search.GenerateAuxiliaryData/SqlExporter.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.Data; using System.Data.SqlClient; @@ -11,7 +12,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -20,14 +20,19 @@ public abstract class SqlExporter : Exporter { private static Assembly _executingAssembly = Assembly.GetExecutingAssembly(); private static string _assemblyName = _executingAssembly.GetName().Name; - - public ISqlConnectionFactory ConnectionFactory { get; } - public SqlExporter(ILogger logger, ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultName) + private Func> OpenSqlConnectionAsync { get; } + + public SqlExporter( + Func> openSqlConnectionAsync, + ILogger logger, + CloudBlobContainer defaultDestinationContainer, + string defaultName) : base(logger, defaultDestinationContainer, defaultName) { _logger = logger; - ConnectionFactory = connectionFactory; + + OpenSqlConnectionAsync = openSqlConnectionAsync; } protected static string GetEmbeddedSqlScript(string resourceName) @@ -38,12 +43,12 @@ protected static string GetEmbeddedSqlScript(string resourceName) public override async Task ExportAsync() { - _logger.LogInformation("Generating {ReportName} report from {DataSource}/{InitialCatalog}.", - _name, ConnectionFactory.DataSource, ConnectionFactory.InitialCatalog); - JContainer result; - using (var connection = await ConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync()) { + _logger.LogInformation("Generating {ReportName} report from {DataSource}/{InitialCatalog}.", + _name, connection.DataSource, connection.Database); + result = GetResultOfQuery(connection); } diff --git a/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs b/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs index 20acab67b..04e3bb334 100644 --- a/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs +++ b/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs @@ -4,10 +4,10 @@ using System; using System.Data; using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -17,12 +17,12 @@ internal sealed class VerifiedPackagesExporter : SqlExporter private readonly string _verifiedPackagesScript; public VerifiedPackagesExporter( + Func> openGallerySqlConnectionAsync, ILogger logger, - ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultVerifiedPackagesScript, string defaultName) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + : base(openGallerySqlConnectionAsync, logger, defaultDestinationContainer, defaultName) { _verifiedPackagesScript = defaultVerifiedPackagesScript; } diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs index 976b2e208..ee4e750b5 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs +++ b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs @@ -11,8 +11,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using IPackageIdGroup = System.Linq.IGrouping; namespace Stats.AggregateCdnDownloadsInGallery @@ -55,20 +53,13 @@ GROUP BY Stats.[PackageRegistrationKey] DROP TABLE #AggregateCdnDownloadsInGallery"; private const string _storedProcedureName = "[dbo].[SelectTotalDownloadCountsPerPackageVersion]"; - private ISqlConnectionFactory _statisticsDbConnectionFactory; - private ISqlConnectionFactory _galleryDbConnectionFactory; private int _batchSize; private int _batchSleepSeconds; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); - - var galleryDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DestinationDatabase); - _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.DestinationDatabase); _batchSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSize) ?? _defaultBatchSize; _batchSleepSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSleepSeconds) ?? _defaultBatchSleepSeconds; @@ -79,12 +70,14 @@ public override async Task Run() // Gather download counts data from statistics warehouse IReadOnlyList downloadData; Logger.LogInformation("Using batch size {BatchSize} and batch sleep seconds {BatchSleepSeconds}.", _batchSize, _batchSleepSeconds); - Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog); var stopwatch = Stopwatch.StartNew(); - using (var statisticsDatabase = await _statisticsDbConnectionFactory.CreateAsync()) + using (var statisticsDatabase = await OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase)) using (var statisticsDatabaseTransaction = statisticsDatabase.BeginTransaction(IsolationLevel.Snapshot)) { + Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", + statisticsDatabase.DataSource, statisticsDatabase.Database); + downloadData = ( await statisticsDatabase.QueryWithRetryAsync( _storedProcedureName, @@ -106,7 +99,7 @@ await statisticsDatabase.QueryWithRetryAsync( return; } - using (var destinationDatabase = await _galleryDbConnectionFactory.CreateAsync()) + using (var destinationDatabase = await OpenSqlConnectionAsync(JobArgumentNames.DestinationDatabase)) { // Fetch package registrations so we can match package ID to package registration key. var packageRegistrationLookup = await GetPackageRegistrations(destinationDatabase); diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj index 3d2de7505..66c03503c 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj +++ b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj @@ -75,9 +75,6 @@ 9.0.1 - - 2.25.0-master-30453 - diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs index 85ae20a62..e7a4bad21 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs @@ -10,7 +10,6 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; using NuGet.Versioning; namespace Stats.CreateAzureCdnWarehouseReports @@ -22,25 +21,28 @@ public class DownloadCountReport private readonly TimeSpan _defaultCommandTimeout = TimeSpan.FromMinutes(30); internal const string ReportName = "downloads.v1.json"; + private Func> OpenStatisticsSqlConnectionAsync { get; } + public DownloadCountReport( + Func> openStatisticsSqlConnectionAsync, ILogger logger, - IEnumerable targets, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) - : base(logger, targets, statisticsDbConnectionFactory, galleryDbConnectionFactory) + IEnumerable targets) + : base(logger, targets) { + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection downloadData; - _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", + connection.DataSource, connection.Database); + downloadData = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs index 7411526f1..588daf973 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs @@ -11,7 +11,6 @@ using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -22,14 +21,14 @@ public class DownloadsPerToolVersionReport private readonly TimeSpan _defaultCommandTimeout = TimeSpan.FromMinutes(30); internal const string ReportName = "tools.v1.json"; + private Func> OpenStatisticsSqlConnectionAsync { get; } + public DownloadsPerToolVersionReport( + Func> openStatisticsSqlConnectionAsync, ILogger logger, CloudStorageAccount cloudStorageAccount, - string statisticsContainerName, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) - : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, - statisticsDbConnectionFactory, galleryDbConnectionFactory) + string statisticsContainerName) + : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }) { } @@ -37,12 +36,13 @@ public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection data; - _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", + connection.DataSource, connection.Database); + data = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs index cdccbb09d..e32a5bd1c 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -23,27 +22,33 @@ public class GalleryTotalsReport (SELECT COUNT([Key]) FROM Packages WITH (NOLOCK) WHERE Listed = 1 AND Deleted = 0) AS TotalPackages"; internal const string ReportName = "stats-totals.json"; + private Func> OpenGallerySqlConnectionAsync { get; } + + private Func> OpenStatisticsSqlConnectionAsync { get; } + public GalleryTotalsReport( + Func> openGallerySqlConnectionAsync, + Func> openStatisticsSqlConnectionAsync, ILogger logger, CloudStorageAccount cloudStorageAccount, - string statisticsContainerName, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) - : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, - statisticsDbConnectionFactory, galleryDbConnectionFactory) + string statisticsContainerName) + : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }) { + OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync; + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task Run() { // gather package numbers from gallery database GalleryTotalsData totalsData; - _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", - GalleryDbConnectionFactory.DataSource, GalleryDbConnectionFactory.InitialCatalog); - using (var connection = await GalleryDbConnectionFactory.CreateAsync()) + using (var connection = await OpenGallerySqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", + connection.DataSource, connection.Database); + totalsData = (await connection.QueryWithRetryAsync( GalleryQuery, commandType: CommandType.Text, transaction: transaction)).First(); } @@ -52,12 +57,13 @@ public async Task Run() _logger.LogInformation("Unique packages: {UniquePackagesCount}", totalsData.UniquePackages); // gather download count data from statistics warehouse - _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", + connection.DataSource, connection.Database); + totalsData.Downloads = (await connection.ExecuteScalarWithRetryAsync( WarehouseStoredProcedureName, commandType: CommandType.StoredProcedure, diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs index 611390958..4f8f55254 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs @@ -4,14 +4,13 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stopwatch = System.Diagnostics.Stopwatch; namespace Stats.CreateAzureCdnWarehouseReports @@ -25,13 +24,13 @@ public class Job private CloudStorageAccount _cloudStorageAccount; private CloudStorageAccount _dataStorageAccount; private string _statisticsContainerName; - private ISqlConnectionFactory _statisticsDbConnectionFactory; - private ISqlConnectionFactory _galleryDbConnectionFactory; private string _reportName; private string[] _dataContainerNames; private int _sqlCommandTimeoutSeconds = DefaultSqlCommandTimeoutSeconds; private int _perPackageReportDegreeOfParallelism = DefaultPerPackageReportDegreeOfParallelism; + private SqlConnectionStringBuilder StatisticsDatabase { get; set; } + private static readonly IDictionary _storedProcedures = new Dictionary { {ReportNames.NuGetClientVersion, "[dbo].[DownloadReportNuGetClientVersion]" }, @@ -50,14 +49,10 @@ public class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - _sqlCommandTimeoutSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds; - - var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDatabaseConnectionString, secretInjector); + StatisticsDatabase = RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.SourceDatabase); - var galleryDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase); - _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDatabaseConnectionString, secretInjector); + _sqlCommandTimeoutSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds; var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); var dataStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); @@ -78,12 +73,24 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenStatisticsSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); + } + + public Task OpenGallerySqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.SourceDatabase); + } + public override async Task Run() { var reportGenerationTime = DateTime.UtcNow; var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName); - Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", + StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog, + _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); var reportBuilderLogger = LoggerFactory.CreateLogger(); var reportCollectorLogger = LoggerFactory.CreateLogger(); @@ -93,12 +100,12 @@ public override async Task Run() // generate all reports var reportGenerators = new Dictionary { - { new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) } + { new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], _sqlCommandTimeoutSeconds) } }; foreach (var reportGenerator in reportGenerators) @@ -114,12 +121,14 @@ public override async Task Run() { // generate only the specific report var reportBuilder = new ReportBuilder(reportBuilderLogger, _reportName); - var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[_reportName], _sqlCommandTimeoutSeconds); await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); } - Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", + StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog, + _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); // totals reports var stopwatch = Stopwatch.StartNew(); @@ -131,7 +140,7 @@ public override async Task Run() { targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName)); } - var downloadCountReport = new DownloadCountReport(LoggerFactory.CreateLogger(), targets, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var downloadCountReport = new DownloadCountReport(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), targets); await downloadCountReport.Run(); stopwatch.Stop(); @@ -140,7 +149,7 @@ public override async Task Run() stopwatch.Restart(); // build stats-totals.json - var galleryTotalsReport = new GalleryTotalsReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var galleryTotalsReport = new GalleryTotalsReport(OpenGallerySqlConnectionAsync, OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName); await galleryTotalsReport.Run(); stopwatch.Stop(); @@ -149,7 +158,7 @@ public override async Task Run() // build tools.v1.json - var toolsReport = new DownloadsPerToolVersionReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var toolsReport = new DownloadsPerToolVersionReport(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName); await toolsReport.Run(); stopwatch.Stop(); @@ -174,14 +183,14 @@ private static async Task ProcessReport(ILoggerFactory loggerFactory, CloudBlobC private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { - var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(LoggerFactory.CreateLogger(), _statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); + var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), reportGenerationTime, _sqlCommandTimeoutSeconds); if (!dirtyPackageIds.Any()) return; // first process the top 100 packages var top100 = dirtyPackageIds.Take(100); - var reportDataCollector = new ReportDataCollector(LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _sqlCommandTimeoutSeconds); var top100Task = Parallel.ForEach(top100, new ParallelOptions { MaxDegreeOfParallelism = _perPackageReportDegreeOfParallelism }, dirtyPackageId => { var packageId = dirtyPackageId.PackageId.ToLowerInvariant(); @@ -209,9 +218,9 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector( + OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], - _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) } }; @@ -228,7 +237,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer if (top100Task.IsCompleted) { var runToCursor = dirtyPackageIds.First().RunToCuror; - await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDbConnectionFactory, runToCursor, _sqlCommandTimeoutSeconds); + await ReportDataCollector.UpdateDirtyPackageIdCursor(OpenStatisticsSqlConnectionAsync, runToCursor, _sqlCommandTimeoutSeconds); } } } @@ -236,7 +245,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { Logger.LogDebug("Getting list of inactive packages."); - var packageIds = await ReportDataCollector.ListInactivePackageIdReports(_statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); + var packageIds = await ReportDataCollector.ListInactivePackageIdReports(OpenStatisticsSqlConnectionAsync, reportGenerationTime, _sqlCommandTimeoutSeconds); Logger.LogInformation("Found {InactivePackageCount} inactive packages.", packageIds.Count); // Collect the list of reports diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs index faef0cce4..121fa1ed5 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs @@ -8,7 +8,6 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -18,20 +17,12 @@ public abstract class ReportBase protected readonly IReadOnlyCollection Targets; - protected readonly ISqlConnectionFactory StatisticsDbConnectionFactory; - - protected ISqlConnectionFactory GalleryDbConnectionFactory; - protected ReportBase( ILogger logger, - IEnumerable targets, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) + IEnumerable targets) { _logger = logger; Targets = targets.ToList().AsReadOnly(); - StatisticsDbConnectionFactory = statisticsDbConnectionFactory; - GalleryDbConnectionFactory = galleryDbConnectionFactory; } protected async Task GetBlobContainer(StorageContainerTarget target) diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs index be58465b4..3f4b4a71d 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs @@ -8,7 +8,6 @@ using System.Diagnostics; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -16,19 +15,20 @@ internal class ReportDataCollector { private int _commandTimeoutSeconds; private readonly string _procedureName; - private readonly ISqlConnectionFactory _sourceDbConnectionFactory; private ILogger _logger; + private Func> OpenStatisticsSqlConnectionAsync { get; } + public ReportDataCollector( + Func> openStatisticsSqlConnectionAsync, ILogger logger, string procedureName, - ISqlConnectionFactory sourceDbConnectionFactory, int timeout) { + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; _logger = logger; _procedureName = procedureName; - _sourceDbConnectionFactory = sourceDbConnectionFactory; _commandTimeoutSeconds = timeout; } @@ -47,8 +47,8 @@ public async Task CollectAsync(DateTime reportGenerationTime, params } public static async Task> GetDirtyPackageIds( + Func> openStatisticsSqlConnectionAsync, ILogger logger, - ISqlConnectionFactory sourceDbConnectionFactory, DateTime reportGenerationTime, int commandTimeout) { @@ -57,7 +57,8 @@ public static async Task> GetDirtyPackageIds IReadOnlyCollection packageIds = new List(); // Get the data - await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(sourceDbConnectionFactory, reportGenerationTime, commandTimeout), logger); + await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse( + openStatisticsSqlConnectionAsync, reportGenerationTime, commandTimeout), logger); logger.LogInformation("Found {DirtyPackagesCount} dirty packages to update.", packageIds.Count); @@ -65,11 +66,11 @@ public static async Task> GetDirtyPackageIds } public static async Task> ListInactivePackageIdReports( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openStatisticsSqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openStatisticsSqlConnectionAsync()) { var command = new SqlCommand("[dbo].[DownloadReportListInactive]", connection); command.CommandType = CommandType.StoredProcedure; @@ -122,7 +123,7 @@ private static async Task WithRetry(Func action, ILogger logger) private async Task ExecuteSql(DateTime reportGenerationTime, params Tuple[] parameters) { - using (var connection = await _sourceDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { var command = new SqlCommand(_procedureName, connection); command.CommandType = CommandType.StoredProcedure; @@ -146,11 +147,11 @@ private async Task ExecuteSql(DateTime reportGenerationTime, params T } private static async Task> GetDirtyPackageIdsFromWarehouse( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openStatisticsSqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openStatisticsSqlConnectionAsync()) { var command = new SqlCommand("[dbo].[GetDirtyPackageIds]", connection); command.CommandType = CommandType.StoredProcedure; @@ -172,11 +173,11 @@ private static async Task> GetDirtyPackageId } public static async Task UpdateDirtyPackageIdCursor( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openStatisticsSqlConnectionAsync, DateTime runToCursor, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openStatisticsSqlConnectionAsync()) { var command = new SqlCommand("[dbo].[UpdateDirtyPackageIdCursor]", connection); command.CommandType = CommandType.StoredProcedure; diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj index 0b09db277..5181875fb 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj +++ b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj @@ -99,9 +99,6 @@ 2.25.0 - - 2.25.0-master-30453 - 4.3.0-preview1-2524 diff --git a/src/Stats.ImportAzureCdnStatistics/DataImporter.cs b/src/Stats.ImportAzureCdnStatistics/DataImporter.cs index 8f0a9251b..fbe92005c 100644 --- a/src/Stats.ImportAzureCdnStatistics/DataImporter.cs +++ b/src/Stats.ImportAzureCdnStatistics/DataImporter.cs @@ -1,21 +1,22 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Data; using System.Data.SqlClient; using System.Threading.Tasks; -using NuGet.Services.Sql; namespace Stats.ImportAzureCdnStatistics { internal class DataImporter { - private readonly ISqlConnectionFactory _statisticsDbConnectionFactory; private const string _sqlSelectTop1FromTable = "SELECT TOP 1 * FROM [dbo].[{0}]"; - public DataImporter(ISqlConnectionFactory statisticsDbConnectionFactory) + public Func> OpenStatisticsSqlConnectionAsync { get; } + + public DataImporter(Func> openStatisticsSqlConnectionAsync) { - _statisticsDbConnectionFactory = statisticsDbConnectionFactory; + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task GetDataTableAsync(string tableName) @@ -23,7 +24,7 @@ public async Task GetDataTableAsync(string tableName) var dataTable = new DataTable(); var query = string.Format(_sqlSelectTop1FromTable, tableName); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { var tableAdapter = new SqlDataAdapter(query, connection) { diff --git a/src/Stats.ImportAzureCdnStatistics/Job.cs b/src/Stats.ImportAzureCdnStatistics/Job.cs index 6b0b18119..5f82c0762 100644 --- a/src/Stats.ImportAzureCdnStatistics/Job.cs +++ b/src/Stats.ImportAzureCdnStatistics/Job.cs @@ -10,9 +10,8 @@ using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stats.AzureCdnLogs.Common; +using System.Data.SqlClient; namespace Stats.ImportAzureCdnStatistics { @@ -23,16 +22,13 @@ public class Job private string _azureCdnAccountNumber; private string _cloudStorageContainerName; private AzureCdnPlatform _azureCdnPlatform; - private ISqlConnectionFactory _statisticsDbConnectionFactory; private CloudStorageAccount _cloudStorageAccount; private CloudBlobClient _cloudBlobClient; private LogFileProvider _blobLeaseManager; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform); var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); @@ -54,6 +50,11 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenStatisticsSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); + } + public override async Task Run() { // Get the target blob container (for archiving decompressed log files) @@ -65,7 +66,7 @@ public override async Task Run() await deadLetterBlobContainer.CreateIfNotExistsAsync(); // Create a parser - var warehouse = new Warehouse(LoggerFactory, _statisticsDbConnectionFactory); + var warehouse = new Warehouse(OpenStatisticsSqlConnectionAsync, LoggerFactory); var statisticsBlobContainerUtility = new StatisticsBlobContainerUtility( targetBlobContainer, deadLetterBlobContainer, diff --git a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj index 6ba8e18cd..47ace8b1a 100644 --- a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj +++ b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj @@ -125,9 +125,6 @@ 2.25.0 - - 2.25.0-master-30453 - 4.3.0-preview1-2524 diff --git a/src/Stats.ImportAzureCdnStatistics/Warehouse.cs b/src/Stats.ImportAzureCdnStatistics/Warehouse.cs index 6e4c7bcae..e84718fc1 100644 --- a/src/Stats.ImportAzureCdnStatistics/Warehouse.cs +++ b/src/Stats.ImportAzureCdnStatistics/Warehouse.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; using Stats.AzureCdnLogs.Common; using Stopwatch = System.Diagnostics.Stopwatch; @@ -21,7 +20,6 @@ internal class Warehouse private const int _maxRetryCount = 3; private readonly TimeSpan _retryDelay = TimeSpan.FromSeconds(5); private readonly ILogger _logger; - private readonly ISqlConnectionFactory _statisticsDbConnectionFactory; private readonly IDictionary _cachedPackageDimensions = new Dictionary(); private readonly IList _cachedToolDimensions = new List(); private readonly IDictionary _cachedClientDimensions = new Dictionary(); @@ -31,15 +29,17 @@ internal class Warehouse private readonly IDictionary _cachedIpAddressFacts = new Dictionary(); private IReadOnlyCollection _times; - public Warehouse(ILoggerFactory loggerFactory, ISqlConnectionFactory statisticsDbConnectionFactory) + private Func> OpenStatisticsSqlConnectionAsync { get; } + + public Warehouse(Func> openStatisticsSqlConnectionAsync, ILoggerFactory loggerFactory) { if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; _logger = loggerFactory.CreateLogger(); - _statisticsDbConnectionFactory = statisticsDbConnectionFactory ?? throw new ArgumentNullException(nameof(statisticsDbConnectionFactory)); } public async Task InsertDownloadFactsAsync(DataTable downloadFactsDataTable, string logFileName) @@ -47,7 +47,7 @@ public async Task InsertDownloadFactsAsync(DataTable downloadFactsDataTable, str _logger.LogDebug("Inserting into facts table..."); var stopwatch = Stopwatch.StartNew(); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { try @@ -124,7 +124,7 @@ public async Task CreateAsync(IReadOnlyCollection var packages = packagesTask.Result; // create facts data rows by linking source data with dimensions - var dataImporter = new DataImporter(_statisticsDbConnectionFactory); + var dataImporter = new DataImporter(OpenStatisticsSqlConnectionAsync); var factsDataTable = await dataImporter.GetDataTableAsync("Fact_Download"); var knownOperationsAvailable = operations.Any(); @@ -245,7 +245,7 @@ public async Task CreateAsync(IReadOnlyCollection sou var ipAddresses = ipAddressesTask.Result; // create facts data rows by linking source data with dimensions - var dataImporter = new DataImporter(_statisticsDbConnectionFactory); + var dataImporter = new DataImporter(OpenStatisticsSqlConnectionAsync); var dataTable = await dataImporter.GetDataTableAsync("Fact_Dist_Download"); var knownClientsAvailable = clients.Any(); @@ -341,7 +341,7 @@ public async Task StoreLogFileAggregatesAsync(LogFileAggregates logFileAggregate { _logger.LogDebug("Storing log file aggregates..."); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { try { @@ -375,7 +375,7 @@ public async Task> GetAlreadyAggregatedLogFilesAsync _logger.LogDebug("Retrieving already processed log files..."); var alreadyAggregatedLogFiles = new List(); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { try { @@ -433,7 +433,7 @@ private async Task HasImportedStatisticsAsync(string logFileName, string c try { - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { var command = connection.CreateCommand(); command.CommandText = commandText; @@ -474,7 +474,7 @@ private async Task> GetDimension(string dimension, stri _logger.LogDebug("Beginning to retrieve dimension '{Dimension}'.", dimension); IDictionary dimensions; - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { dimensions = await retrieve(connection); } @@ -546,7 +546,7 @@ private async Task> GetDimension(string dimension, str _logger.LogDebug("Beginning to retrieve dimension '{Dimension}'.", dimension); IReadOnlyCollection dimensions; - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { dimensions = await retrieve(connection); } diff --git a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs index a10b412c3..5bc3fb1b7 100644 --- a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs +++ b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs @@ -8,23 +8,18 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stats.ImportAzureCdnStatistics; namespace Stats.RefreshClientDimension { public class RefreshClientDimensionJob : JobBase { - private static ISqlConnectionFactory _statisticsDbConnectionFactory; private static string _targetClientName; private static string _userAgentFilter; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _targetClientName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, "TargetClientName"); _userAgentFilter = JobConfigurationManager.TryGetArgument(jobArgsDictionary, "UserAgentFilter"); @@ -32,7 +27,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary> linkedUserAgents; diff --git a/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj b/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj index 980174366..1de4656f1 100644 --- a/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj +++ b/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj @@ -83,9 +83,6 @@ 9.0.1 - - 2.25.0-master-30453 - 1.2.0 diff --git a/src/Stats.RollUpDownloadFacts/Job.cs b/src/Stats.RollUpDownloadFacts/Job.cs index 76cd61979..51d3dcc40 100644 --- a/src/Stats.RollUpDownloadFacts/Job.cs +++ b/src/Stats.RollUpDownloadFacts/Job.cs @@ -9,8 +9,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Stats.RollUpDownloadFacts { @@ -21,13 +19,10 @@ public class Job private const string _endTemplateFactDownloadDeletion = " records from [dbo].[Fact_Download]"; private const int DefaultMinAgeInDays = 43; private static int _minAgeInDays; - private static ISqlConnectionFactory _statisticsDbConnectionFactory; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _minAgeInDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MinAgeInDays) ?? DefaultMinAgeInDays; Logger.LogInformation("Min age in days: {MinAgeInDays}", _minAgeInDays); @@ -35,7 +30,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary 2.25.0 - - 2.25.0-master-30453 - diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs index 0a3fb2fad..d012fc1de 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs +++ b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs @@ -14,8 +14,6 @@ using Newtonsoft.Json.Linq; using Newtonsoft.Json.Schema; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace UpdateLicenseReports { @@ -41,7 +39,6 @@ internal class Job : JobBase private Uri _licenseReportService; private string _licenseReportUser; private string _licenseReportPassword; - private ISqlConnectionFactory _packageDbConnectionFactory; private int? _retryCount; private NetworkCredential _licenseReportCredentials; @@ -61,9 +58,7 @@ private static PackageLicenseReport CreateReport(JObject messageEvent) public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var dbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - _packageDbConnectionFactory = new AzureSqlConnectionFactory(dbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); var retryCountString = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.RetryCount); if (string.IsNullOrEmpty(retryCountString)) @@ -111,12 +106,12 @@ public override async Task Run() private async Task FetchNextReportUrlAsync() { - Logger.LogInformation("Fetching next report URL from {DataSource}/{InitialCatalog}", - _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); - Uri nextLicenseReport = null; - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) { + Logger.LogInformation("Fetching next report URL from {DataSource}/{InitialCatalog}", + connection.DataSource, connection.Database); + var nextReportUrl = (await connection.QueryAsync( @"SELECT TOP 1 NextLicenseReport FROM GallerySettings")).SingleOrDefault(); @@ -130,11 +125,11 @@ private async Task FetchNextReportUrlAsync() } nextLicenseReport = nextLicenseReport ?? _licenseReportService; - } - Logger.LogInformation("Fetched next report URL '{NextReportUrl}' from {DataSource}/{InitialCatalog}", - (nextLicenseReport == null ? string.Empty : nextLicenseReport.AbsoluteUri), - _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + Logger.LogInformation("Fetched next report URL '{NextReportUrl}' from {DataSource}/{InitialCatalog}", + (nextLicenseReport == null ? string.Empty : nextLicenseReport.AbsoluteUri), + connection.DataSource, connection.Database); + } return nextLicenseReport; } @@ -242,7 +237,7 @@ private async Task ProcessReportsAsync(Uri nextLicenseReport) Logger.LogInformation("Storing next license report URL: {NextReportUrl}", nextLicenseReport.AbsoluteUri); // Record the next report to the database so we can check it again if we get aborted before finishing. - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) { await connection.QueryAsync(@" UPDATE GallerySettings @@ -274,7 +269,7 @@ UPDATE GallerySettings private async Task StoreReportAsync(PackageLicenseReport report) { - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) using (var command = connection.CreateCommand()) { command.CommandText = "AddPackageLicenseReport2"; diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.csproj b/src/UpdateLicenseReports/UpdateLicenseReports.csproj index 0d9b7210c..6a824cd81 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.csproj +++ b/src/UpdateLicenseReports/UpdateLicenseReports.csproj @@ -74,9 +74,6 @@ 2.0.10 - - 2.25.0-master-30453 - diff --git a/src/Validation.Common.Job/JsonConfigurationJob.cs b/src/Validation.Common.Job/JsonConfigurationJob.cs index 9355f050c..7cc47941f 100644 --- a/src/Validation.Common.Job/JsonConfigurationJob.cs +++ b/src/Validation.Common.Job/JsonConfigurationJob.cs @@ -4,12 +4,10 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.Common; using System.IO; using System.Net; using System.Net.Http; using System.Reflection; -using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; using Microsoft.ApplicationInsights; @@ -22,7 +20,6 @@ using NuGet.Services.KeyVault; using NuGet.Services.Logging; using NuGet.Services.ServiceBus; -using NuGet.Services.Sql; using NuGet.Services.Validation; using NuGetGallery; using NuGetGallery.Diagnostics; @@ -64,6 +61,9 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(_serviceProvider); + RegisterDatabase(_serviceProvider); + ServicePointManager.DefaultConnectionLimit = MaximumConnectionsPerServer; } @@ -109,15 +109,6 @@ private IServiceProvider GetServiceProvider(IConfigurationRoot configurationRoot return new AutofacServiceProvider(containerBuilder.Build()); } - protected virtual DbConnection CreateDbConnection(IServiceProvider serviceProvider) where T : IDbConfiguration - { - var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; - var connectionFactory = new AzureSqlConnectionFactory(connectionString, - serviceProvider.GetRequiredService()); - - return Task.Run(() => connectionFactory.CreateAsync()).Result; - } - private void ConfigureDefaultJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { services.Configure(configurationRoot.GetSection(GalleryDbConfigurationSectionName)); @@ -142,12 +133,12 @@ private void ConfigureDefaultJobServices(IServiceCollection services, IConfigura services.AddScoped(p => { - return new ValidationEntitiesContext(CreateDbConnection(p)); + return new ValidationEntitiesContext(CreateSqlConnection()); }); services.AddScoped(p => { - return new EntitiesContext(CreateDbConnection(p), readOnly: true); + return new EntitiesContext(CreateSqlConnection(), readOnly: true); }); services.AddTransient(p => diff --git a/src/Validation.Common.Job/Validation.Common.Job.csproj b/src/Validation.Common.Job/Validation.Common.Job.csproj index 4b4fa7e6a..15b0f761e 100644 --- a/src/Validation.Common.Job/Validation.Common.Job.csproj +++ b/src/Validation.Common.Job/Validation.Common.Job.csproj @@ -94,25 +94,22 @@ 4.8.0-preview4.5289 - 2.26.0-master-34394 + 2.27.0 - 2.26.0-master-34394 + 2.27.0 - 2.26.0-master-34736 - - - 2.26.0-master-34394 + 2.27.0 - 2.26.0-master-34394 + 2.27.0 - 2.26.0-master-34736 + 2.27.0 - 4.4.5-dev-34648 + 4.4.5-dev-35356 2.5.0 @@ -128,7 +125,9 @@ - + + Designer + diff --git a/src/Validation.Common.Job/Validation.Common.Job.nuspec b/src/Validation.Common.Job/Validation.Common.Job.nuspec index f01f8d6cd..1b70aaa35 100644 --- a/src/Validation.Common.Job/Validation.Common.Job.nuspec +++ b/src/Validation.Common.Job/Validation.Common.Job.nuspec @@ -16,13 +16,13 @@ - - - - - - - + + + + + + + diff --git a/src/Validation.Common/Validation.Common.csproj b/src/Validation.Common/Validation.Common.csproj index 716e67d13..75cc170e2 100644 --- a/src/Validation.Common/Validation.Common.csproj +++ b/src/Validation.Common/Validation.Common.csproj @@ -108,10 +108,10 @@ 4.1.0 - 2.26.0-master-33196 + 2.27.0 - 2.25.0 + 2.27.0 3.2.0 diff --git a/src/Validation.PackageSigning.ProcessSignature/Job.cs b/src/Validation.PackageSigning.ProcessSignature/Job.cs index 16997f935..14f9e5bdc 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Job.cs +++ b/src/Validation.PackageSigning.ProcessSignature/Job.cs @@ -35,7 +35,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddScoped(p => { - return new EntitiesContext(CreateDbConnection(p), readOnly: false); + return new EntitiesContext(CreateSqlConnection(), readOnly: false); }); services.Add(ServiceDescriptor.Transient(typeof(IEntityRepository<>), typeof(EntityRepository<>))); diff --git a/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json b/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json index 6e6021b58..d195c2287 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json +++ b/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json @@ -19,7 +19,9 @@ "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$" }, "ProcessSignature": { - "AllowedRepositorySigningCertificates": [], + "AllowedRepositorySigningCertificates": [ + "cf7ac17ad047ecd5fdc36822031b12d4ef078b6f2b4c5e6ba41f8ff2cf4bad67" + ], "V3ServiceIndexUrl": "https://api.nuget.org/v3/index.json" }, diff --git a/src/Validation.PackageSigning.ProcessSignature/SignatureValidator.cs b/src/Validation.PackageSigning.ProcessSignature/SignatureValidator.cs index 9016dba69..57b3c872a 100644 --- a/src/Validation.PackageSigning.ProcessSignature/SignatureValidator.cs +++ b/src/Validation.PackageSigning.ProcessSignature/SignatureValidator.cs @@ -420,7 +420,7 @@ private async Task IsValidRepositorySignatureAsync(Context context, T s context.Message.PackageId, context.Message.PackageVersion, context.Message.ValidationId, - signature.V3ServiceIndexUrl.AbsoluteUri); + signature.V3ServiceIndexUrl?.AbsoluteUri); return false; } diff --git a/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj b/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj index a27ed8411..1ae9d197a 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj +++ b/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj @@ -85,11 +85,6 @@ Validation.PackageSigning.Core - - - 2.26.0-master-34394 - - ..\..\build diff --git a/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj b/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj index c5bc64caa..6c215c062 100644 --- a/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj +++ b/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj @@ -55,7 +55,7 @@ 1.1.2 - 2.26.0-master-34736 + 2.27.0 diff --git a/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs index 261143584..8594f9cda 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs @@ -18,6 +18,18 @@ public class InitializationManagerFacts { public class TheInitializeAsyncMethod : FactsBase { + [Fact] + public async Task ThrowsIfAlreadyInitialized() + { + // Arrange + _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); + + // Act & Assert + var e = await Assert.ThrowsAsync(() => _target.InitializeAsync()); + + Assert.Equal("Attempted to initialize the revalidation job when it is already initialized!", e.Message); + } + [Fact] public async Task RemovesPreviousRevalidations() { @@ -32,7 +44,7 @@ public async Task RemovesPreviousRevalidations() var firstRemove = true; - _revalidationState.Setup(s => s.RemoveRevalidationsAsync(1000)) + _packageState.Setup(s => s.RemovePackageRevalidationsAsync(1000)) .ReturnsAsync(() => { if (firstRemove) @@ -47,7 +59,7 @@ public async Task RemovesPreviousRevalidations() // Act & assert await _target.InitializeAsync(); - _revalidationState.Verify(s => s.RemoveRevalidationsAsync(1000), Times.Exactly(2)); + _packageState.Verify(s => s.RemovePackageRevalidationsAsync(1000), Times.Exactly(2)); } [Fact] @@ -71,13 +83,12 @@ public async Task InsertsMicrosoftThenPreinstalledThenDependenciesThenAllOtherPa remainingPackages: new[] { new Package { PackageRegistrationKey = 3, DownloadCount = 20, NormalizedVersion = "3.0.0", }, - }); // Act & assert await _target.InitializeAsync(); - _revalidationState.Verify( + _packageState.Verify( s => s.AddPackageRevalidationsAsync(It.IsAny>()), Times.Exactly(4)); @@ -116,7 +127,7 @@ public async Task AddsRevalidationsByDescendingOrderOfDownloadCounts() // Act & assert await _target.InitializeAsync(); - _revalidationState.Verify( + _packageState.Verify( s => s.AddPackageRevalidationsAsync(It.IsAny>()), Times.Once); @@ -153,7 +164,7 @@ public async Task InsertsPackageVersionsByDescendingOrder() // Act & assert await _target.InitializeAsync(); - _revalidationState.Verify( + _packageState.Verify( s => s.AddPackageRevalidationsAsync(It.IsAny>()), Times.Once); @@ -209,7 +220,7 @@ public async Task PartitionsPackagesIntoBatchesOf1000OrLessVersions(int[] packag // Act & assert await _target.InitializeAsync(); - _revalidationState.Verify( + _packageState.Verify( s => s.AddPackageRevalidationsAsync(It.IsAny>()), Times.Exactly(expectedBatches)); } @@ -248,7 +259,36 @@ public static IEnumerable PartitionsPackagesIntoBatchesOf1000OrLessVer }; } - // Partitions packages by batches of 1000 versions (with intelligent weights) + [Fact] + public async Task MarksAsInitializedAfterAddingRevalidations() + { + // Arrange + Setup(remainingPackages: new[] + { + new Package { PackageRegistrationKey = 3, DownloadCount = 20, NormalizedVersion = "3.0.0", }, + }); + + int order = 0; + int addRevalidationOrder = 0; + int markAsInitializedOrder = 0; + + _packageState + .Setup(s => s.AddPackageRevalidationsAsync(It.IsAny>())) + .Callback(() => addRevalidationOrder = order++) + .Returns(Task.CompletedTask); + + _settings + .Setup(s => s.MarkAsInitializedAsync()) + .Callback(() => markAsInitializedOrder = order++) + .Returns(Task.CompletedTask); + + // Act & Assert + await _target.InitializeAsync(); + + _settings.Verify(s => s.MarkAsInitializedAsync(), Times.Once); + + Assert.True(markAsInitializedOrder > addRevalidationOrder); + } private void Setup( IEnumerable microsoftPackages = null, @@ -344,7 +384,7 @@ public TheInitializeAsyncMethod() { _revalidationBatches = new List>(); - _revalidationState + _packageState .Setup(s => s.AddPackageRevalidationsAsync(It.IsAny>())) .Callback>(r => _revalidationBatches.Add(r)) .Returns(Task.CompletedTask); @@ -353,20 +393,34 @@ public TheInitializeAsyncMethod() public class TheVerifyAsyncMethod : FactsBase { + [Fact] + public async Task ThrowsIfNotInitialized() + { + _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(false); + + var e = await Assert.ThrowsAsync(() => _target.VerifyInitializationAsync()); + + Assert.Equal("Expected revalidation state to be initialized", e.Message); + } + [Fact] public async Task ThrowsIfAppropriatePackageCountDoesNotMatchRevalidationCount() { + _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); _packageFinder.Setup(f => f.AppropriatePackageCount()).Returns(100); - _revalidationState.Setup(s => s.PackageRevalidationCountAsync()).ReturnsAsync(50); + _packageState.Setup(s => s.PackageRevalidationCountAsync()).ReturnsAsync(50); + + var e = await Assert.ThrowsAsync(() => _target.VerifyInitializationAsync()); - var exception = await Assert.ThrowsAsync(async () => await _target.VerifyInitializationAsync()); + Assert.Equal("Expected 100 revalidation, found 50", e.Message); } [Fact] public async Task DoesNotThrowIfCountsMatch() { + _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); _packageFinder.Setup(f => f.AppropriatePackageCount()).Returns(100); - _revalidationState.Setup(s => s.PackageRevalidationCountAsync()).ReturnsAsync(100); + _packageState.Setup(s => s.PackageRevalidationCountAsync()).ReturnsAsync(100); await _target.VerifyInitializationAsync(); } @@ -374,7 +428,8 @@ public async Task DoesNotThrowIfCountsMatch() public class FactsBase { - public readonly Mock _revalidationState; + public readonly Mock _settings; + public readonly Mock _packageState; public readonly Mock _packageFinder; public readonly InitializationConfiguration _config; @@ -382,13 +437,15 @@ public class FactsBase public FactsBase() { - _revalidationState = new Mock(); + _settings = new Mock(); + _packageState = new Mock(); _packageFinder = new Mock(); _config = new InitializationConfiguration(); _target = new InitializationManager( - _revalidationState.Object, + _settings.Object, + _packageState.Object, _packageFinder.Object, _config, Mock.Of>()); diff --git a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj index 0fe5be2eb..172a9d5c3 100644 --- a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj +++ b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj @@ -66,9 +66,11 @@ - + + + @@ -80,5 +82,8 @@ Tests.ContextHelpers + + + \ No newline at end of file diff --git a/tests/NuGet.Services.Revalidate.Tests/RevalidationStateServiceFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs similarity index 69% rename from tests/NuGet.Services.Revalidate.Tests/RevalidationStateServiceFacts.cs rename to tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs index 3afff254a..69e976fb4 100644 --- a/tests/NuGet.Services.Revalidate.Tests/RevalidationStateServiceFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -10,9 +11,9 @@ using Tests.ContextHelpers; using Xunit; -namespace NuGet.Services.Revalidate.Tests +namespace NuGet.Services.Revalidate.Tests.Services { - public class RevalidationStateServiceFacts + public class PackageRevalidationStateServiceFacts { public class TheAddPackageRevalidationsAsyncMethod : FactsBase { @@ -50,7 +51,7 @@ public async Task RemovesRevalidations() }); // Act & Assert - var result = await _target.RemoveRevalidationsAsync(5); + var result = await _target.RemovePackageRevalidationsAsync(5); Assert.Equal(2, result); Assert.Equal(0, _context.Object.PackageRevalidations.Count()); @@ -69,7 +70,7 @@ public async Task RespectsMaxParameter() }); // Act & Assert - var result = await _target.RemoveRevalidationsAsync(1); + var result = await _target.RemovePackageRevalidationsAsync(1); Assert.Equal(1, result); Assert.Equal(1, _context.Object.PackageRevalidations.Count()); @@ -93,18 +94,37 @@ public async Task ReturnsRevalidationCount() } } + public class TheCountRevalidationsEnqueuedInPastHourAsyncMethod : FactsBase + { + [Fact] + public async Task ReturnsRevalidationCount() + { + var now = DateTime.UtcNow; + + _context.Mock(packageRevalidations: new List + { + new PackageRevalidation { PackageId = "A", Enqueued = now.Subtract(TimeSpan.FromDays(4)) }, + new PackageRevalidation { PackageId = "B", Enqueued = now.Subtract(TimeSpan.FromHours(3)) }, + new PackageRevalidation { PackageId = "C", Enqueued = now.Subtract(TimeSpan.FromMinutes(2)) }, + new PackageRevalidation { PackageId = "D", Enqueued = now.Subtract(TimeSpan.FromSeconds(1)) }, + }); + + Assert.Equal(2, await _target.CountRevalidationsEnqueuedInPastHourAsync()); + } + } + public class FactsBase { public readonly Mock _context; - public readonly RevalidationStateService _target; + public readonly PackageRevalidationStateService _target; public FactsBase() { _context = new Mock(); - _target = new RevalidationStateService( + _target = new PackageRevalidationStateService( _context.Object, - Mock.Of>()); + Mock.Of>()); } } } diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs new file mode 100644 index 000000000..7403364bf --- /dev/null +++ b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs @@ -0,0 +1,273 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NuGetGallery; +using Xunit; + +namespace NuGet.Services.Revalidate.Tests.Services +{ + public class RevalidationJobStateServiceFacts + { + public class TheIsInitializedAsyncMethod : FactsBase + { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReturnsInitializedValue(bool isInitialized) + { + _state + .Setup(s => s.GetStateAsync()) + .ReturnsAsync(new RevalidationState + { + IsInitialized = isInitialized + }); + + Assert.Equal(isInitialized, await _target.IsInitializedAsync()); + + _state.Verify(s => s.GetStateAsync(), Times.Once); + } + } + + public class TheMarkAsInitializedAsyncMethod : FactsBase + { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UpdatesState(bool isInitialized) + { + var result = new RevalidationState + { + IsInitialized = isInitialized + }; + + _state + .Setup(s => s.UpdateStateAsync(It.IsAny>())) + .Callback((Action a) => a(result)) + .Returns(Task.CompletedTask); + + await _target.MarkAsInitializedAsync(); + + _state.Verify( + s => s.UpdateStateAsync(It.IsAny>()), + Times.Once); + + Assert.True(result.IsInitialized); + } + } + + public class TheIsKillswitchActiveAsyncMethod : FactsBase + { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReturnsIsKillswitchActiveValue(bool isKillswitchActive) + { + _state + .Setup(s => s.GetStateAsync()) + .ReturnsAsync(new RevalidationState + { + IsKillswitchActive = isKillswitchActive + }); + + Assert.Equal(isKillswitchActive, await _target.IsKillswitchActiveAsync()); + + _state.Verify(s => s.GetStateAsync(), Times.Once); + } + } + + public class TheGetDesiredPackageEventRateAsyncMethod : FactsBase + { + [Fact] + public async Task ReturnsDesiredPackageEventRateValue() + { + // Arrange + bool? result = null; + var state = new RevalidationState + { + DesiredPackageEventRate = 123 + }; + + _state + .Setup(s => s.MaybeUpdateStateAsync(It.IsAny>())) + .Callback((Func a) => + { + result = a(state); + }) + .ReturnsAsync(state); + + // Act & Assert + Assert.Equal(123, await _target.GetDesiredPackageEventRateAsync()); + Assert.False(result); + + _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny>()), Times.Once); + } + + [Theory] + [InlineData(0, 100)] + [InlineData(600, 500)] + public async Task UpdatesRateIfCurrentRateOutsideofConfiguredBounds(int storedRate, int expectedRate) + { + // Arrange + bool? result = null; + var state = new RevalidationState + { + DesiredPackageEventRate = storedRate + }; + + _state + .Setup(s => s.MaybeUpdateStateAsync(It.IsAny>())) + .Callback((Func a) => + { + result = a(state); + }) + .ReturnsAsync(state); + + // Act & Assert + Assert.Equal(expectedRate, await _target.GetDesiredPackageEventRateAsync()); + Assert.True(result); + + _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny>()), Times.Once); + } + } + + public class TheResetDesiredPackageEventRateAsyncMethod : FactsBase + { + [Fact] + public async Task UpdatesState() + { + var result = new RevalidationState + { + DesiredPackageEventRate = 123 + }; + + _state + .Setup(s => s.UpdateStateAsync(It.IsAny>())) + .Callback((Action a) => a(result)) + .Returns(Task.CompletedTask); + + await _target.ResetDesiredPackageEventRateAsync(); + + _state.Verify( + s => s.UpdateStateAsync(It.IsAny>()), + Times.Once); + + Assert.Equal(100, result.DesiredPackageEventRate); + } + } + + public class TheIncreaseDesiredPackageEventRateAsyncMethod : FactsBase + { + [Fact] + public async Task IncreasesDesiredPackageEventRateValue() + { + // Arrange + bool? result = null; + var state = new RevalidationState + { + DesiredPackageEventRate = 123 + }; + + _state + .Setup(s => s.MaybeUpdateStateAsync(It.IsAny>())) + .Callback((Func a) => + { + result = a(state); + }) + .ReturnsAsync(state); + + // Act + await _target.IncreaseDesiredPackageEventRateAsync(); + + // Assert + Assert.Equal(124, state.DesiredPackageEventRate); + Assert.True(result); + + _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny>()), Times.Once); + } + + [Fact] + public async Task DoesntIncreaseIfReachedConfiguredMaxRate() + { + // Arrange + bool? result = null; + var state = new RevalidationState + { + DesiredPackageEventRate = 500 + }; + + _state + .Setup(s => s.MaybeUpdateStateAsync(It.IsAny>())) + .Callback((Func a) => + { + result = a(state); + }) + .ReturnsAsync(state); + + // Act + await _target.IncreaseDesiredPackageEventRateAsync(); + + // Assert + Assert.Equal(500, state.DesiredPackageEventRate); + Assert.False(result); + + _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny>()), Times.Once); + } + + [Fact] + public async Task ResetsToConfiguredMaxRateIfPastConfiguredMax() + { + // Arrange + bool? result = null; + var state = new RevalidationState + { + DesiredPackageEventRate = 600 + }; + + _state + .Setup(s => s.MaybeUpdateStateAsync(It.IsAny>())) + .Callback((Func a) => + { + result = a(state); + }) + .ReturnsAsync(state); + + // Act + await _target.IncreaseDesiredPackageEventRateAsync(); + + // Assert + Assert.Equal(500, state.DesiredPackageEventRate); + Assert.True(result); + + _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny>()), Times.Once); + } + } + + public class FactsBase + { + protected readonly Mock _state; + protected readonly RevalidationConfiguration _config; + + protected readonly RevalidationJobStateService _target; + + public FactsBase() + { + _state = new Mock(); + + _config = new RevalidationConfiguration + { + MinPackageEventRate = 100, + MaxPackageEventRate = 500, + }; + + _target = new RevalidationJobStateService( + _state.Object, + _config, + Mock.Of>()); + } + } + } +} diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationServiceFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationServiceFacts.cs index 839fea7ff..b4d72744c 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationServiceFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationServiceFacts.cs @@ -2,9 +2,11 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Moq; +using NuGet.Services.Logging; using NuGet.Services.Validation; using Xunit; @@ -14,6 +16,16 @@ public class RevalidationServiceFacts { public class TheRunAsyncMethod : FactsBase { + [Fact] + public async Task ThrowsIfNotInitialized() + { + Setup(isInitialized: false); + + var e = await Assert.ThrowsAsync(() => _target.RunAsync()); + + Assert.Equal("The revalidation service must be initialized before running revalidations", e.Message); + } + [Fact] public async Task ReturnsUnrecoverableError() { @@ -84,12 +96,24 @@ public async Task IfNotSingleton_ReturnsUnrecoverableError() var result = await _target.StartNextRevalidationAsync(); _singletonService.Verify(s => s.IsSingletonAsync(), Times.Once); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Never); _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); Assert.Equal(RevalidationResult.UnrecoverableError, result); } + [Fact] + public async Task IfSingletonCheckThrows_ReturnsRetryLater() + { + // Arrange + Setup(singletonThrows: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + [Fact] public async Task IfKillswitchActive_ReturnsRetryLater() { @@ -99,13 +123,25 @@ public async Task IfKillswitchActive_ReturnsRetryLater() // Act & Assert var result = await _target.StartNextRevalidationAsync(); - _stateService.Verify(s => s.IsKillswitchActiveAsync(), Times.Once); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _jobState.Verify(s => s.IsKillswitchActiveAsync(), Times.Once); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Never); _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); Assert.Equal(RevalidationResult.RetryLater, result); } + [Fact] + public async Task IfKillswitchThrows_ReturnsRetryLater() + { + // Arrange + Setup(killswitchThrows: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + [Fact] public async Task IfThrottled_ReturnsRetryLater() { @@ -116,12 +152,24 @@ public async Task IfThrottled_ReturnsRetryLater() var result = await _target.StartNextRevalidationAsync(); _throttler.Verify(s => s.IsThrottledAsync(), Times.Once); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Never); _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); Assert.Equal(RevalidationResult.RetryLater, result); } + [Fact] + public async Task IfThrottledThrows_ReturnsRetryLater() + { + // Arrange + Setup(throttledThrows: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + [Fact] public async Task IfUnhealthy_ReturnsRetryLater() { @@ -131,13 +179,25 @@ public async Task IfUnhealthy_ReturnsRetryLater() // Act & Assert var result = await _target.StartNextRevalidationAsync(); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Never); _healthService.Verify(h => h.IsHealthyAsync(), Times.Once); _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); Assert.Equal(RevalidationResult.RetryLater, result); } + [Fact] + public async Task IfUnhealthyThrows_ReturnsRetryLater() + { + // Arrange + Setup(healthyThrows: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + [Fact] public async Task IfRevalidationQueueEmpty_ReturnsRetryLater() { @@ -148,17 +208,29 @@ public async Task IfRevalidationQueueEmpty_ReturnsRetryLater() var result = await _target.StartNextRevalidationAsync(); _singletonService.Verify(s => s.IsSingletonAsync(), Times.Once); - _stateService.Verify(s => s.IsKillswitchActiveAsync(), Times.Exactly(2)); + _jobState.Verify(s => s.IsKillswitchActiveAsync(), Times.Exactly(2)); _throttler.Verify(s => s.IsThrottledAsync(), Times.Once); _healthService.Verify(h => h.IsHealthyAsync(), Times.Once); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Once); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Once); _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); Assert.Equal(RevalidationResult.RetryLater, result); } + [Fact] + public async Task IfRevalidationQueueNextThrows_ReturnsRetryLater() + { + // Arrange + Setup(nextThrows: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + [Fact] public async Task StartsNextRevalidation() { @@ -174,8 +246,8 @@ public async Task StartsNextRevalidation() .Callback(() => enqueueStep = order++) .Returns(Task.CompletedTask); - _stateService - .Setup(s => s.MarkRevalidationAsEnqueuedAsync(It.IsAny())) + _packageState + .Setup(s => s.MarkPackageRevalidationAsEnqueuedAsync(It.IsAny())) .Callback(() => markStep = order++) .Returns(Task.CompletedTask); @@ -184,11 +256,11 @@ public async Task StartsNextRevalidation() // Assert _singletonService.Verify(s => s.IsSingletonAsync(), Times.Once); - _stateService.Verify(s => s.IsKillswitchActiveAsync(), Times.Exactly(2)); + _jobState.Verify(s => s.IsKillswitchActiveAsync(), Times.Exactly(2)); _throttler.Verify(s => s.IsThrottledAsync(), Times.Once); _healthService.Verify(h => h.IsHealthyAsync(), Times.Once); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Once); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Once); _validationEnqueuer.Verify( e => e.StartValidationAsync(It.Is(m => @@ -197,7 +269,7 @@ public async Task StartsNextRevalidation() m.ValidationTrackingId == _revalidation.ValidationTrackingId.Value)), Times.Once); - _stateService.Verify(s => s.MarkRevalidationAsEnqueuedAsync(_revalidation), Times.Once); + _packageState.Verify(s => s.MarkPackageRevalidationAsEnqueuedAsync(_revalidation), Times.Once); _telemetryService.Verify(t => t.TrackPackageRevalidationStarted(_revalidation.PackageId, _revalidation.PackageNormalizedVersion)); Assert.Equal(RevalidationResult.RevalidationEnqueued, result); @@ -208,7 +280,8 @@ public async Task StartsNextRevalidation() public class FactsBase { - protected readonly Mock _stateService; + protected readonly Mock _jobState; + protected readonly Mock _packageState; protected readonly Mock _singletonService; protected readonly Mock _throttler; protected readonly Mock _healthService; @@ -223,7 +296,8 @@ public class FactsBase public FactsBase() { - _stateService = new Mock(); + _jobState = new Mock(); + _packageState = new Mock(); _singletonService = new Mock(); _throttler = new Mock(); _healthService = new Mock(); @@ -231,6 +305,14 @@ public FactsBase() _validationEnqueuer = new Mock(); _telemetryService = new Mock(); + _telemetryService + .Setup(t => t.TrackStartNextRevalidationOperation()) + .Returns(new DurationMetric( + Mock.Of(), + "Name", + new StartNextRevalidationOperation(), + Mock.Of>>())); + _config = new RevalidationConfiguration { ShutdownWaitInterval = TimeSpan.MinValue, @@ -244,7 +326,8 @@ public FactsBase() }; _target = new RevalidationService( - _stateService.Object, + _jobState.Object, + _packageState.Object, _singletonService.Object, _throttler.Object, _healthService.Object, @@ -255,13 +338,35 @@ public FactsBase() Mock.Of>()); } - protected void Setup(bool isSingleton = true, bool killswitchActive = false, bool isThrottled = false, bool isHealthy = true, PackageRevalidation next = null) + protected void Setup( + bool isInitialized = true, + bool isSingleton = true, + bool killswitchActive = false, + bool isThrottled = false, + bool isHealthy = true, + PackageRevalidation next = null, + bool initializedThrows = false, + bool singletonThrows = false, + bool killswitchThrows = false, + bool throttledThrows = false, + bool healthyThrows = false, + bool nextThrows = false) { + _jobState.Setup(s => s.IsInitializedAsync()).ReturnsAsync(isInitialized); _singletonService.Setup(s => s.IsSingletonAsync()).ReturnsAsync(isSingleton); - _stateService.Setup(s => s.IsKillswitchActiveAsync()).ReturnsAsync(killswitchActive); + _jobState.Setup(s => s.IsKillswitchActiveAsync()).ReturnsAsync(killswitchActive); _throttler.Setup(t => t.IsThrottledAsync()).ReturnsAsync(isThrottled); _healthService.Setup(t => t.IsHealthyAsync()).ReturnsAsync(isHealthy); _revalidationQueue.Setup(q => q.NextOrNullAsync()).ReturnsAsync(next); + + var exception = new Exception(); + + if (initializedThrows) _jobState.Setup(s => s.IsInitializedAsync()).ThrowsAsync(exception); + if (singletonThrows) _singletonService.Setup(s => s.IsSingletonAsync()).ThrowsAsync(exception); + if (killswitchThrows) _jobState.Setup(s => s.IsKillswitchActiveAsync()).ThrowsAsync(exception); + if (throttledThrows) _throttler.Setup(t => t.IsThrottledAsync()).ThrowsAsync(exception); + if (healthyThrows) _healthService.Setup(t => t.IsHealthyAsync()).ThrowsAsync(exception); + if (nextThrows) _revalidationQueue.Setup(q => q.NextOrNullAsync()).ThrowsAsync(exception); } protected void SetupUnrecoverableErrorResult() diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs new file mode 100644 index 000000000..7b5da8254 --- /dev/null +++ b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs @@ -0,0 +1,64 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace NuGet.Services.Revalidate.Tests.Services +{ + public class RevalidationThrottlerFacts + { + public class TheIsThrottledAsyncMethod + { + private readonly Mock _settings; + private readonly Mock _state; + private readonly RevalidationConfiguration _config; + + private readonly IRevalidationThrottler _target; + + public TheIsThrottledAsyncMethod() + { + _settings = new Mock(); + _state = new Mock(); + + _config = new RevalidationConfiguration(); + + _target = new RevalidationThrottler( + _settings.Object, + _state.Object, + _config, + Mock.Of>()); + } + + [Fact] + public async Task ReturnsTrueIfRecentRevalidationsMoreThanDesiredRate() + { + // Arrange + _settings.Setup(s => s.GetDesiredPackageEventRateAsync()).ReturnsAsync(50); + _state.Setup(s => s.CountRevalidationsEnqueuedInPastHourAsync()).ReturnsAsync(100); + + // Act & Assert + Assert.True(await _target.IsThrottledAsync()); + + _settings.Verify(s => s.GetDesiredPackageEventRateAsync(), Times.Once); + _state.Verify(s => s.CountRevalidationsEnqueuedInPastHourAsync(), Times.Once); + } + + [Fact] + public async Task ReturnsFalseIfRecentRevalidationsLessThanDesiredRate() + { + // Arrange + _settings.Setup(s => s.GetDesiredPackageEventRateAsync()).ReturnsAsync(100); + _state.Setup(s => s.CountRevalidationsEnqueuedInPastHourAsync()).ReturnsAsync(50); + + // Act & Assert + Assert.False(await _target.IsThrottledAsync()); + + _settings.Verify(s => s.GetDesiredPackageEventRateAsync(), Times.Once); + _state.Verify(s => s.CountRevalidationsEnqueuedInPastHourAsync(), Times.Once); + } + } + } +} diff --git a/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs b/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs index 9a202115a..9e5cb51d7 100644 --- a/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs +++ b/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs @@ -3,11 +3,12 @@ using System; using System.Data; +using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; -using NuGet.Services.Sql; using Search.GenerateAuxiliaryData; using Xunit; @@ -44,11 +45,16 @@ public void GetRankings_ReturnsRankings() private static RankingsExporter CreateExporter() { return new RankingsExporter( + DoNotOpenSqlConnectionAsync, new LoggerFactory().CreateLogger(), - connectionFactory: new Mock().Object, defaultDestinationContainer: new CloudBlobContainer(new Uri("https://nuget.org")), defaultRankingsScript: "b", defaultName: "c"); } + + public static Task DoNotOpenSqlConnectionAsync() + { + return Task.FromResult((SqlConnection)null); + } } } \ No newline at end of file diff --git a/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs b/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs index 50b8aea23..c31f36116 100644 --- a/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs +++ b/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs @@ -3,11 +3,12 @@ using System; using System.Data; +using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; -using NuGet.Services.Sql; using Search.GenerateAuxiliaryData; using Xunit; @@ -43,11 +44,16 @@ public void GetVerifiedPackagesReturnsJsonString() private static VerifiedPackagesExporter CreateExporter() { return new VerifiedPackagesExporter( + DoNotOpenSqlConnectionAsync, new LoggerFactory().CreateLogger(), - connectionFactory: new Mock().Object, defaultDestinationContainer: new CloudBlobContainer(new Uri("https://nuget.org")), defaultVerifiedPackagesScript: "b", defaultName: "c"); } + + public static Task DoNotOpenSqlConnectionAsync() + { + return Task.FromResult((SqlConnection)null); + } } } \ No newline at end of file diff --git a/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj b/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj index 510501a59..b0662efcb 100644 --- a/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj +++ b/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj @@ -75,5 +75,8 @@ 2.3.1 + + + \ No newline at end of file