Merge pull request #475 from NuGet/dev

[ReleasePrep][2018.07.06] RI of dev into master

src/NuGet.Services.Validation.Orchestrator/Job.cs

@@ -66,6 +66,7 @@ public class Job : JobBase
         private const string PackageSignatureBindingKey = PackageSigningSectionName;
         private const string PackageCertificatesBindingKey = PackageCertificatesSectionName;
         private const string ScanAndSignBindingKey = ScanAndSignSectionName;
+        private const string ScanBindingKey = "Scan";
         private const string ValidationStorageBindingKey = "ValidationStorage";
         private const string OrchestratorBindingKey = "Orchestrator";
 
@@ -362,6 +363,7 @@ private static IServiceProvider CreateProvider(IServiceCollection services)
             ConfigurePackageSigningValidators(containerBuilder);
             ConfigurePackageCertificatesValidator(containerBuilder);
             ConfigureScanAndSignProcessor(containerBuilder);
+            ConfigureScanValidator(containerBuilder);
 
             return new AutofacServiceProvider(containerBuilder.Build());
         }
@@ -468,6 +470,21 @@ private static void ConfigureScanAndSignProcessor(ContainerBuilder builder)
                 .AsSelf();
         }
 
+        private static void ConfigureScanValidator(ContainerBuilder builder)
+        {
+            builder
+                .RegisterType<ValidatorStateService>()
+                .WithParameter(
+                    (pi, ctx) => pi.ParameterType == typeof(string),
+                    (pi, ctx) => ValidatorName.ScanOnly)
+                .Keyed<IValidatorStateService>(ScanBindingKey);
+
+            builder
+                .RegisterType<ScanValidator>()
+                .WithKeyedParameter(typeof(IValidatorStateService), ScanBindingKey)
+                .AsSelf();
+        }
+
         private T GetRequiredService<T>()
         {
             return _serviceProvider.GetRequiredService<T>();

src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj

@@ -51,6 +51,7 @@
     <Compile Include="DiskMailSender.cs" />
     <Compile Include="Configuration\EmailConfiguration.cs" />
     <Compile Include="Error.cs" />
+    <Compile Include="PackageSigning\Scan\ScanValidator.cs" />
     <Compile Include="Services\IEntityService.cs" />
     <Compile Include="IMessageService.cs" />
     <Compile Include="IPackageStatusProcessor.cs" />

src/NuGet.Services.Validation.Orchestrator/PackageSigning/Scan/ScanValidator.cs

@@ -0,0 +1,129 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using NuGet.Jobs.Validation;
+using NuGet.Jobs.Validation.PackageSigning.Storage;
+using NuGet.Jobs.Validation.Storage;
+using NuGet.Jobs.Validation.ScanAndSign;
+using NuGet.Services.Validation.Vcs;
+using NuGetGallery;
+
+namespace NuGet.Services.Validation.Orchestrator.PackageSigning.ScanAndSign
+{
+    [ValidatorName(ValidatorName.ScanOnly)]
+    public class ScanValidator : BaseValidator, IValidator
+    {
+        private readonly IValidationEntitiesContext _validationContext;
+        private readonly IValidatorStateService _validatorStateService;
+        private readonly ICorePackageService _packageService;
+        private readonly ICriteriaEvaluator<Package> _criteriaEvaluator;
+        private readonly IScanAndSignEnqueuer _scanAndSignEnqueuer;
+        private readonly ScanAndSignConfiguration _configuration;
+        private readonly ILogger<ScanAndSignProcessor> _logger;
+
+        public ScanValidator(
+            IValidationEntitiesContext validationContext,
+            IValidatorStateService validatorStateService,
+            ICorePackageService packageService,
+            ICriteriaEvaluator<Package> criteriaEvaluator,
+            IScanAndSignEnqueuer scanAndSignEnqueuer,
+            IOptionsSnapshot<ScanAndSignConfiguration> configurationAccessor,
+            ILogger<ScanAndSignProcessor> logger)
+        {
+            _validationContext = validationContext ?? throw new ArgumentNullException(nameof(validationContext));
+            _validatorStateService = validatorStateService ?? throw new ArgumentNullException(nameof(validatorStateService));
+            _packageService = packageService ?? throw new ArgumentNullException(nameof(packageService));
+            _criteriaEvaluator = criteriaEvaluator ?? throw new ArgumentNullException(nameof(criteriaEvaluator));
+            _scanAndSignEnqueuer = scanAndSignEnqueuer ?? throw new ArgumentNullException(nameof(scanAndSignEnqueuer));
+
+            if (configurationAccessor == null)
+            {
+                throw new ArgumentNullException(nameof(configurationAccessor));
+            }
+            if (configurationAccessor.Value == null)
+            {
+                throw new ArgumentException($"{nameof(configurationAccessor.Value)} property is null", nameof(configurationAccessor));
+            }
+            _configuration = configurationAccessor.Value;
+
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+
+            configurationAccessor = configurationAccessor ?? throw new ArgumentNullException(nameof(configurationAccessor));
+
+            if (configurationAccessor.Value == null)
+            {
+                throw new ArgumentException($"{nameof(configurationAccessor.Value)} property is null", nameof(configurationAccessor));
+            }
+
+            _configuration = configurationAccessor.Value;
+        }
+
+        public async Task<IValidationResult> GetResultAsync(IValidationRequest request)
+        {
+            if (request == null)
+            {
+                throw new ArgumentNullException(nameof(request));
+            }
+
+            var validatorStatus = await _validatorStateService.GetStatusAsync(request);
+
+            return validatorStatus.ToValidationResult();
+        }
+
+        public async Task<IValidationResult> StartAsync(IValidationRequest request)
+        {
+            if (request == null)
+            {
+                throw new ArgumentNullException(nameof(request));
+            }
+
+            var validatorStatus = await _validatorStateService.GetStatusAsync(request);
+
+            if (validatorStatus.State != ValidationStatus.NotStarted)
+            {
+                _logger.LogWarning(
+                    "Scan only validation with validation Id {ValidationId} ({PackageId} {PackageVersion}) has already started.",
+                    request.ValidationId,
+                    request.PackageId,
+                    request.PackageVersion);
+
+                return validatorStatus.ToValidationResult();
+            }
+
+            if (ShouldSkipScan(request))
+            {
+                return ValidationResult.Succeeded;
+            }
+
+            await _scanAndSignEnqueuer.EnqueueScanAsync(request.ValidationId, request.NupkgUrl);
+
+            var result = await _validatorStateService.TryAddValidatorStatusAsync(request, validatorStatus, ValidationStatus.Incomplete);
+
+            return result.ToValidationResult();
+        }
+
+        private bool ShouldSkipScan(IValidationRequest request)
+        {
+            var package = _packageService.FindPackageByIdAndVersionStrict(
+                request.PackageId,
+                request.PackageVersion);
+
+            if (!_criteriaEvaluator.IsMatch(_configuration.PackageCriteria, package))
+            {
+                _logger.LogInformation(
+                    "The scan for {ValidationId} ({PackageId} {PackageVersion}) was skipped due to package criteria configuration.",
+                    request.ValidationId,
+                    request.PackageId,
+                    request.PackageVersion);
+
+                return true;
+            }
+
+            return false;
+        }
+    }
+}

src/NuGet.Services.Validation.Orchestrator/PackageSigning/ScanAndSign/ScanAndSignProcessor.cs

@@ -79,6 +79,17 @@ public async Task CleanUpAsync(IValidationRequest request)
                 return;
             }
 
+            if (!_configuration.RepositorySigningEnabled)
+            {
+                _logger.LogWarning(
+                    "Skipping cleanup of .nupkg for validation ID {ValidationId} ({PackageId} {PackageVersion})",
+                    request.ValidationId,
+                    request.PackageId,
+                    request.PackageVersion);
+
+                return;
+            }
+
             _logger.LogInformation(
                 "Cleaning up the .nupkg URL for validation ID {ValidationId} ({PackageId} {PackageVersion}).",
                 request.ValidationId,
@@ -96,9 +107,9 @@ public async Task<IValidationResult> GetResultAsync(IValidationRequest request)
                 throw new ArgumentNullException(nameof(request));
             }
 
-            var validatorStatus = await _validatorStateService.GetStatusAsync(request);
+            var result = await GetProcessorStatusAsync(request);
 
-            return validatorStatus.ToValidationResult();
+            return result.ToValidationResult();
         }
 
         public async Task<IValidationResult> StartAsync(IValidationRequest request)
@@ -108,17 +119,17 @@ public async Task<IValidationResult> StartAsync(IValidationRequest request)
                 throw new ArgumentNullException(nameof(request));
             }
 
-            var validatorStatus = await _validatorStateService.GetStatusAsync(request);
+            var processorStatus = await GetProcessorStatusAsync(request);
 
-            if (validatorStatus.State != ValidationStatus.NotStarted)
+            if (processorStatus.State != ValidationStatus.NotStarted)
             {
                 _logger.LogWarning(
                     "Scan and Sign validation with validation Id {ValidationId} ({PackageId} {PackageVersion}) has already started.",
                     request.ValidationId,
                     request.PackageId,
                     request.PackageVersion);
 
-                return validatorStatus.ToValidationResult();
+                return processorStatus.ToValidationResult();
             }
 
             if (await ShouldRepositorySignAsync(request))
@@ -144,11 +155,29 @@ public async Task<IValidationResult> StartAsync(IValidationRequest request)
                 await _scanAndSignEnqueuer.EnqueueScanAsync(request.ValidationId, request.NupkgUrl);
             }
 
-            var result = await _validatorStateService.TryAddValidatorStatusAsync(request, validatorStatus, ValidationStatus.Incomplete);
+            var result = await _validatorStateService.TryAddValidatorStatusAsync(request, processorStatus, ValidationStatus.Incomplete);
 
             return result.ToValidationResult();
         }
 
+        private async Task<ValidatorStatus> GetProcessorStatusAsync(IValidationRequest request)
+        {
+            var validatorStatus = await _validatorStateService.GetStatusAsync(request);
+
+            if (!_configuration.RepositorySigningEnabled && validatorStatus.NupkgUrl != null)
+            {
+                _logger.LogWarning(
+                    "Suppressing .nupkg url as repository signing is disabled for {ValidationId} ({PackageId} {PackageVersion})",
+                    request.ValidationId,
+                    request.PackageId,
+                    request.PackageVersion);
+
+                validatorStatus.NupkgUrl = null;
+            }
+
+            return validatorStatus;
+        }
+
         private bool ShouldSkipScan(IValidationRequest request)
         {
             var package = _packageService.FindPackageByIdAndVersionStrict(
@@ -171,13 +200,6 @@ private bool ShouldSkipScan(IValidationRequest request)
 
         private async Task<bool> ShouldRepositorySignAsync(IValidationRequest request)
         {
-            if (!_configuration.RepositorySigningEnabled)
-            {
-                _logger.LogInformation("Repository signing is disabed. Scanning instead of signing package");
-
-                return false;
-            }
-
             var hasRepositorySignature = await _validationContext
                 .PackageSignatures
                 .Where(s => s.PackageKey == request.PackageKey)

src/Validation.Common.Job/Validation/ValidatorName.cs

@@ -8,6 +8,7 @@ public static class ValidatorName
         public const string Vcs = "VcsValidator";
         public const string PackageCertificate = "PackageCertificatesValidator";
         public const string ScanAndSign = "ScanAndSign";
+        public const string ScanOnly = "ScanOnly";
         public const string PackageSignatureProcessor = "PackageSigningValidator";
         public const string PackageSignatureValidator = "PackageSigningValidator2";
     }

src/Validation.PackageSigning.ProcessSignature/Settings/dev.json

@@ -19,7 +19,7 @@
   },
   "ProcessSignature": {
     "AllowedRepositorySigningCertificates": [
-      "0e5f38f57dc1bcc806d8494f4f90fbcedd988b46760709cbeec6f4219aa6157d"
+      "cf6ce6768ef858a3a667be1af8aa524d386c7f59a34542713f5dfb0d79acf3dd"
     ],
     "V3ServiceIndexUrl": "https://apidev.nugettest.org/v3/index.json"
   },

src/Validation.PackageSigning.ProcessSignature/Settings/int.json

@@ -18,7 +18,9 @@
     "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$"
   },
   "ProcessSignature": {
-    "AllowedRepositorySigningCertificates": [],
+    "AllowedRepositorySigningCertificates": [
+      "cf6ce6768ef858a3a667be1af8aa524d386c7f59a34542713f5dfb0d79acf3dd"
+    ],
     "V3ServiceIndexUrl": "https://apiint.nugettest.org/v3/index.json"
   },
 

tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs

@@ -210,7 +210,7 @@ public async Task AcceptsValidSignedPackage()
             Assert.Empty(result.Issues);
         }
 
-        [Fact]
+        [Fact(Skip = "Flaky")]
         public async Task RejectsUntrustedSigningCertificate()
         {
             // Arrange