Merge pull request #591 from NuGet/dev

[ReleasePrep][2018.10.22]FI of master into dev

src/NuGet.Services.Revalidate/Job.cs

@@ -32,6 +32,7 @@ public class Job : ValidationJobBase
         private const string RebuildPreinstalledSetArgumentName = "RebuildPreinstalledSet";
         private const string InitializeArgumentName = "Initialize";
         private const string VerifyInitializationArgumentName = "VerifyInitialization";
+
         private const string JobConfigurationSectionName = "RevalidateJob";
 
         private static readonly TimeSpan RetryLaterSleepDuration = TimeSpan.FromMinutes(5);
@@ -112,6 +113,7 @@ await scope.ServiceProvider
         protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
         {
             services.Configure<RevalidationConfiguration>(configurationRoot.GetSection(JobConfigurationSectionName));
+
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Initialization);
             services.AddSingleton(provider => provider.GetRequiredService<IOptionsSnapshot<RevalidationConfiguration>>().Value.Health);

src/NuGet.Services.Revalidate/Settings/dev.json

@@ -16,8 +16,8 @@
       "ComponentPath": "NuGet/Package Publishing"
     },
 
-    "MinPackageEventRate": 120,
-    "MaxPackageEventRate": 400,
+    "MinPackageEventRate": "#{Jobs.nuget.services.revalidation.MinPackageEventRate}",
+    "MaxPackageEventRate": "#{Jobs.nuget.services.revalidation.MaxPackageEventRate}",
 
     "RetryLaterSleep": "00:00:30",
 

src/NuGet.Services.Revalidate/Settings/int.json

@@ -16,8 +16,8 @@
       "ComponentPath": "NuGet/Package Publishing"
     },
 
-    "MinPackageEventRate": 120,
-    "MaxPackageEventRate": 400,
+    "MinPackageEventRate": "#{Jobs.nuget.services.revalidation.MinPackageEventRate}",
+    "MaxPackageEventRate": "#{Jobs.nuget.services.revalidation.MaxPackageEventRate}",
 
     "RetryLaterSleep": "00:00:30",
 

src/NuGet.Services.Revalidate/Settings/prod.json

@@ -16,8 +16,8 @@
       "ComponentPath": "NuGet/Package Publishing"
     },
 
-    "MinPackageEventRate": 120,
-    "MaxPackageEventRate": 400,
+    "MinPackageEventRate": "#{Jobs.nuget.services.revalidation.MinPackageEventRate}",
+    "MaxPackageEventRate": "#{Jobs.nuget.services.revalidation.MaxPackageEventRate}",
 
     "RetryLaterSleep": "00:00:30",
 

src/NuGet.Services.Validation.Orchestrator/Services/IEntityService.cs

@@ -17,9 +17,15 @@ public interface IEntityService<T> where T : class, IEntity
         /// </summary>
         /// <param name="id">The id .</param>
         /// <param name="version">The version.</param>
-        /// <returns></returns>
+        /// <returns>The entity.</returns>
         IValidatingEntity<T> FindPackageByIdAndVersionStrict(string id, string version);
 
+        /// <summary>
+        /// Find the entity based on the key.
+        /// </summary>
+        /// <returns>The entity.</returns>
+        IValidatingEntity<T> FindPackageByKey(int key);
+
         /// <summary>
         /// Update the status of the entity.
         /// </summary>
@@ -35,7 +41,7 @@ public interface IEntityService<T> where T : class, IEntity
         /// <param name="entity">The entity.</param>
         /// <param name="metadata">The metadata.</param>
         /// <param name="commitChanges">True if the changes will be commited to the database.</param>
-        /// <returns></returns>
+        /// <returns>A <see cref="Task"/> that can be used to await for the operation completion.</returns>
         Task UpdateMetadataAsync(T entity, object metadata, bool commitChanges = true);
     }
 }

src/NuGet.Services.Validation.Orchestrator/Services/PackageEntityService.cs

@@ -1,6 +1,9 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
+using System.Linq;
+using System.Data.Entity;
 using System.Threading.Tasks;
 using NuGetGallery;
 using NuGetGallery.Packaging;
@@ -13,10 +16,12 @@ namespace NuGet.Services.Validation.Orchestrator
     public class PackageEntityService : IEntityService<Package>
     {
         private ICorePackageService _galleryEntityService;
+        private IEntityRepository<Package> _packageRepository;
 
-        public PackageEntityService(ICorePackageService galleryEntityService)
+        public PackageEntityService(ICorePackageService galleryEntityService, IEntityRepository<Package> packageRepository)
         {
-            _galleryEntityService = galleryEntityService;
+            _galleryEntityService = galleryEntityService ?? throw new ArgumentNullException(nameof(galleryEntityService));
+            _packageRepository = packageRepository ?? throw new ArgumentNullException(nameof(packageRepository));
         }
 
         public IValidatingEntity<Package> FindPackageByIdAndVersionStrict(string id, string version)
@@ -25,6 +30,18 @@ public IValidatingEntity<Package> FindPackageByIdAndVersionStrict(string id, str
             return p == null ? null : new PackageValidatingEntity(p);
         }
 
+        public IValidatingEntity<Package> FindPackageByKey(int key)
+        {
+            var package = _packageRepository
+                   .GetAll()
+                   .Include(p => p.LicenseReports)
+                   .Include(p => p.PackageRegistration)
+                   .Include(p => p.User)
+                   .Include(p => p.SymbolPackages)
+                   .SingleOrDefault(p => p.Key == key);
+            return package == null ? null : new PackageValidatingEntity(package);
+        }
+
         public async Task UpdateStatusAsync(Package entity, PackageStatus newStatus, bool commitChanges = true)
         {
             await _galleryEntityService.UpdatePackageStatusAsync(entity, newStatus, commitChanges);

src/NuGet.Services.Validation.Orchestrator/Services/SymbolEntityService.cs

@@ -13,10 +13,13 @@ namespace NuGet.Services.Validation.Orchestrator
     /// </summary>
     public class SymbolEntityService : IEntityService<SymbolPackage>
     {
-        ICoreSymbolPackageService _galleryEntityService;
-        public SymbolEntityService(ICoreSymbolPackageService galleryEntityService)
+        private ICoreSymbolPackageService _galleryEntityService;
+        private IEntityRepository<SymbolPackage> _symbolsPackageRepository;
+
+        public SymbolEntityService(ICoreSymbolPackageService galleryEntityService, IEntityRepository<SymbolPackage> symbolsPackageRepository)
         {
             _galleryEntityService = galleryEntityService ?? throw new ArgumentNullException(nameof(galleryEntityService));
+            _symbolsPackageRepository = symbolsPackageRepository ?? throw new ArgumentNullException(nameof(symbolsPackageRepository));
         }
 
         /// <summary>
@@ -35,6 +38,14 @@ public IValidatingEntity<SymbolPackage> FindPackageByIdAndVersionStrict(string i
             return symbolPackage == null ? null : new SymbolPackageValidatingEntity(symbolPackage);
         }
 
+        public IValidatingEntity<SymbolPackage> FindPackageByKey(int key)
+        {
+            var symbolPackage = _symbolsPackageRepository
+                   .GetAll()
+                   .SingleOrDefault(p => p.Key == key);
+            return symbolPackage == null ? null : new SymbolPackageValidatingEntity(symbolPackage);
+        }
+
         public async Task UpdateStatusAsync(SymbolPackage entity, PackageStatus newStatus, bool commitChanges = true)
         {
             if(entity == null)

src/NuGet.Services.Validation.Orchestrator/SymbolValidationMessageHandler.cs

@@ -73,7 +73,11 @@ public async Task<bool> HandleAsync(PackageValidationMessageData message)
                 message.PackageNormalizedVersion,
                 message.ValidationTrackingId))
             {
-                var symbolPackageEntity = _gallerySymbolService.FindPackageByIdAndVersionStrict(message.PackageId, message.PackageNormalizedVersion);
+                // When a message is sent from the Gallery with validation of a new entity, the EntityKey will be null because the message is sent to the service bus before the entity is persisted in the DB
+                // However when a revalidation happens or when the message is re-sent by the orchestrator the message will contain the key. In this case the key is used to find the entity to validate.
+                var symbolPackageEntity = message.EntityKey.HasValue
+                    ? _gallerySymbolService.FindPackageByKey(message.EntityKey.Value)
+                    : _gallerySymbolService.FindPackageByIdAndVersionStrict(message.PackageId, message.PackageNormalizedVersion);
 
                 if (symbolPackageEntity == null)
                 {
@@ -94,9 +98,10 @@ public async Task<bool> HandleAsync(PackageValidationMessageData message)
                     }
                     else
                     {
-                        _logger.LogInformation("Could not find symbols for package {PackageId} {PackageNormalizedVersion} in DB, retrying",
+                        _logger.LogInformation("Could not find symbols for package {PackageId} {PackageNormalizedVersion} {Key} in DB, retrying",
                             message.PackageId,
-                            message.PackageNormalizedVersion);
+                            message.PackageNormalizedVersion,
+                            message.EntityKey.HasValue);
 
                         return false;
                     }
@@ -114,6 +119,7 @@ public async Task<bool> HandleAsync(PackageValidationMessageData message)
                 }
 
                 var processorStats = await _validationSetProcessor.ProcessValidationsAsync(validationSet);
+                // As part of the processing the validation outcome the orchestrator will send itself a message if validation are still being processed.
                 await _validationOutcomeProcessor.ProcessValidationOutcomeAsync(validationSet, symbolPackageEntity, processorStats);
             }
 

src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolScanValidator.cs

@@ -113,7 +113,7 @@ private bool ShouldSkipScan(IValidationRequest request)
         {
             var symbolPackage = _symbolPackageService
                 .FindSymbolPackagesByIdAndVersion(request.PackageId,request.PackageVersion)
-                .FirstOrDefault(sp => sp.StatusKey == PackageStatus.Validating);
+                .FirstOrDefault(sp => sp.Key == request.PackageKey);
 
             if (symbolPackage == null)
             {

src/NuGet.Services.Validation.Orchestrator/ValidationOutcomeProcessor.cs

@@ -264,7 +264,12 @@ private async Task ScheduleCheckIfNotTimedOut(PackageValidationSet validationSet
             // Schedule another check if we haven't reached the validation set timeout yet.
             if (validationSetDuration <= _validationConfiguration.TimeoutValidationSetAfter)
             {
-                var messageData = new PackageValidationMessageData(validationSet.PackageId, validationSet.PackageNormalizedVersion, validationSet.ValidationTrackingId, validationSet.ValidatingType);
+                var messageData = new PackageValidationMessageData(
+                    validationSet.PackageId,
+                    validationSet.PackageNormalizedVersion,
+                    validationSet.ValidationTrackingId,
+                    validationSet.ValidatingType,
+                    entityKey: validationSet.PackageKey);
                 var postponeUntil = DateTimeOffset.UtcNow + _validationConfiguration.ValidationMessageRecheckPeriod;
 
                 await _validationEnqueuer.StartValidationAsync(messageData, postponeUntil);

src/Validation.Symbols/ISymbolsFileService.cs

@@ -21,10 +21,9 @@ public interface ISymbolsFileService
         /// <summary>
         /// Downloads the snupkg file.
         /// </summary>
-        /// <param name="packageId">The package id.</param>
-        /// <param name="packageNormalizedVersion">The package normalized version.</param>
+        /// <param name="snupkgUri">The uri of the snupkg.</param>
         /// <param name="cancellationToken">Cancellation token.</param>
-        /// <returns>The nupkg stream.</returns>
-        Task<Stream> DownloadSnupkgFileAsync(string packageId, string packageNormalizedVersion, CancellationToken cancellationToken);
+        /// <returns>The snupkg stream.</returns>
+        Task<Stream> DownloadSnupkgFileAsync(string snupkgUri, CancellationToken cancellationToken);
     }
 }

src/Validation.Symbols/ISymbolsValidatorService.cs

@@ -1,23 +1,21 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-using System.Collections.Generic;
-using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
+using NuGet.Jobs.Validation.Symbols.Core;
 using NuGet.Services.Validation;
 
 namespace Validation.Symbols
 {
     public interface ISymbolsValidatorService
     {
         /// <summary>
-        /// Validates the symbols against the PE files. 
+        /// Validates the symbol package.
         /// </summary>
-        /// <param name="packageId">The package Id.</param>
-        /// <param name="packageNormalizedVersion">The package normalized version.</param>
-        /// <param name="token">A cancellation token to be used for cancellation of the async execution.</param>
-        /// <returns></returns>
-        Task<IValidationResult> ValidateSymbolsAsync(string packageId, string packageNormalizedVersion, CancellationToken token);
+        /// <param name="message">The <see cref="SymbolsValidatorMessage"/> regarding to the symbols pacakge to be validated..</param>
+        /// <param name="token">Cancellation token.</param>
+        /// <returns>The validation result.</returns>
+        Task<IValidationResult> ValidateSymbolsAsync(SymbolsValidatorMessage message, CancellationToken token);
     }
 }

src/Validation.Symbols/Job.cs

@@ -40,11 +40,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
                     configurationAccessor.Value.ValidationPackageConnectionString,
                     readAccessGeoRedundant: false), c.GetRequiredService<IDiagnosticsService>());
 
-                var symbolValidationStorageService = new CloudBlobCoreFileStorageService(new CloudBlobClientWrapper(
-                    configurationAccessor.Value.ValidationSymbolsConnectionString,
-                    readAccessGeoRedundant: false), c.GetRequiredService<IDiagnosticsService>());
-
-                return new SymbolsFileService(packageStorageService, packageValidationStorageService, symbolValidationStorageService);
+                return new SymbolsFileService(packageStorageService, packageValidationStorageService, c.GetRequiredService<IFileDownloader>());
             });
             services.AddSingleton(new TelemetryClient());
         }

src/Validation.Symbols/Settings/dev.json

@@ -13,6 +13,7 @@
     "PackageConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$",
     "ValidationSymbolsConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$"
   },
+  "PackageDownloadTimeout": "00:10:00",
   "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
   "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
   "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",

src/Validation.Symbols/Settings/int.json

@@ -13,7 +13,7 @@
     "PackageConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$",
     "ValidationSymbolsConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$"
   },
-
+  "PackageDownloadTimeout": "00:10:00",
   "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
   "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
   "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",

src/Validation.Symbols/Settings/prod.json

@@ -13,7 +13,7 @@
     "PackageConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$",
     "ValidationSymbolsConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$"
   },
-
+  "PackageDownloadTimeout": "00:10:00",
   "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
   "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
   "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",

src/Validation.Symbols/SymbolsFileService.cs

@@ -6,19 +6,20 @@
 using System.Threading.Tasks;
 using System.Threading;
 using NuGetGallery;
+using NuGet.Jobs.Validation;
 
 namespace Validation.Symbols
 {
     public class SymbolsFileService : ISymbolsFileService
     {
         private CorePackageFileService _packageFileService;
         private CorePackageFileService _packageValidationFileService;
-        private CorePackageFileService _symbolValidationFileService;
+        private IFileDownloader _fileDownloader;
 
         public SymbolsFileService(
             ICoreFileStorageService packageStorageService,
             ICoreFileStorageService packageValidationStorageService,
-            ICoreFileStorageService symbolValidationStorageService)
+            IFileDownloader fileDownloader)
         {
             if (packageStorageService == null)
             {
@@ -28,32 +29,14 @@ public SymbolsFileService(
             {
                 throw new ArgumentNullException(nameof(packageValidationStorageService));
             }
-            if (symbolValidationStorageService == null)
-            {
-                throw new ArgumentNullException(nameof(symbolValidationStorageService));
-            }
             _packageFileService = new CorePackageFileService(packageStorageService, new PackageFileMetadataService());
             _packageValidationFileService = new CorePackageFileService(packageValidationStorageService, new PackageFileMetadataService());
-            _symbolValidationFileService = new CorePackageFileService(symbolValidationStorageService, new SymbolPackageFileMetadataService());
+            _fileDownloader = fileDownloader ?? throw new ArgumentNullException(nameof(fileDownloader));
         }
 
-        public async Task<Stream> DownloadSnupkgFileAsync(string packageId, string packageNormalizedVersion, CancellationToken cancellationToken)
+        public async Task<Stream> DownloadSnupkgFileAsync(string snupkgUri, CancellationToken cancellationToken)
         {
-            var package = new Package()
-            {
-                NormalizedVersion = packageNormalizedVersion,
-                PackageRegistration = new PackageRegistration()
-                {
-                    Id = packageId
-                }
-            };
-
-            if (await _symbolValidationFileService.DoesValidationPackageFileExistAsync(package))
-            {
-                return await _symbolValidationFileService.DownloadValidationPackageFileAsync(package);
-            }
-
-            throw new FileNotFoundException(string.Format("Symbols package {0} {1} not found in the validation container.", packageId, packageNormalizedVersion));
+            return await _fileDownloader.DownloadAsync(new Uri(snupkgUri), cancellationToken);
         }
 
         public async Task<Stream> DownloadNupkgFileAsync(string packageId, string packageNormalizedVersion, CancellationToken cancellationToken)

src/Validation.Symbols/SymbolsValidatorMessageHandler.cs

@@ -86,7 +86,7 @@ public async Task<bool> HandleAsync(SymbolsValidatorMessage message)
                     return true;
                 }
 
-                var validationResult = await _symbolValidatorService.ValidateSymbolsAsync(message.PackageId, message.PackageNormalizedVersion, CancellationToken.None);
+                var validationResult = await _symbolValidatorService.ValidateSymbolsAsync(message, CancellationToken.None);
 
                 if (validationResult.Status == ValidationStatus.Failed || validationResult.Status == ValidationStatus.Succeeded)
                 {