Merge pull request #19 from sensslen/simon/add-rate-limiting-support

.github/workflows/action.yml

@@ -65,6 +65,7 @@ jobs:
     strategy:
       matrix:
         targetFramework: [net6.0, net7.0, net8.0]
+        project: [App, Tests]
 
         include:
           - targetFramework: net6.0
@@ -90,4 +91,14 @@ jobs:
         run: dotnet publish ./src/NuGetUtility/NuGetUtility.csproj --configuration Release -o ./release -f ${{ matrix.targetFramework }} --no-restore
 
       - name: check
-        run: dotnet ./release/NuGetUtility.dll -ji ./.github/workflows/assets/projectsToCheck.json -t -a ./.github/workflows/assets/allowedLicenses.json -o JsonPretty -override ./.github/workflows/assets/overwritePackageInformation.json -ignore ./.github/workflows/assets/ignorePackages.json -mapping ./.github/workflows/assets/urlToLicenseMapping.json
+        run: dotnet ./release/NuGetUtility.dll -ji ./.github/workflows/assets/${{ matrix.project }}/projectsToCheck.json -t -a ./.github/workflows/assets/${{ matrix.project }}/allowedLicenses.json -o JsonPretty -override ./.github/workflows/assets/${{ matrix.project }}/overwritePackageInformation.json -ignore ./.github/workflows/assets/${{ matrix.project }}/ignorePackages.json -mapping ./.github/workflows/assets/${{ matrix.project }}/urlToLicenseMapping.json -d ./licenses/${{ matrix.project }}/${{ matrix.targetFramework }}
+
+      - name: show downloaded licenses
+        shell: pwsh
+        run: |
+          foreach($file in Get-ChildItem -Path ./licenses/${{ matrix.project }}/${{ matrix.targetFramework }})
+          {
+            Write-Host ::group::$file
+            Get-Content $file.FullName
+            Write-Host ::endgroup::
+          }

.github/workflows/assets/App/allowedLicenses.json

@@ -0,0 +1 @@
+["MIT","Apache-2.0","MS-EULA"]

.github/workflows/assets/App/overwritePackageInformation.json

@@ -0,0 +1,2 @@
+[
+]

.github/workflows/assets/App/projectsToCheck.json

@@ -0,0 +1 @@
+["./src/NuGetUtility/NuGetUtility.csproj"]

.github/workflows/assets/App/urlToLicenseMapping.json

@@ -0,0 +1,2 @@
+{
+}

.github/workflows/assets/Tests/ignorePackages.json

@@ -0,0 +1 @@
+["NETStandard.Library"]

.github/workflows/assets/overwritePackageInformation.json → .github/workflows/assets/Tests/overwritePackageInformation.json

@@ -79,4 +79,4 @@
         "Version": "2.1.1",
         "License": "MIT"
     }
-]
+]

.github/workflows/assets/Tests/projectsToCheck.json

@@ -0,0 +1 @@
+["./tests/NuGetUtility.Test/NuGetUtility.Test.csproj"]

.github/workflows/assets/urlToLicenseMapping.json → .github/workflows/assets/Tests/urlToLicenseMapping.json

@@ -1,3 +1,3 @@
 {
     "https://raw.githubusercontent.com/bchavez/Bogus/master/LICENSE": "MIT"
-}
+}

src/NuGetUtility/LicenseValidator/LicenseValidator.cs

@@ -28,7 +28,8 @@ public LicenseValidator(IImmutableDictionary<Uri, string> licenseMapping,
         }
 
         public async Task<IEnumerable<LicenseValidationResult>> Validate(
-            IAsyncEnumerable<ReferencedPackageWithContext> packages)
+            IAsyncEnumerable<ReferencedPackageWithContext> packages,
+            CancellationToken token)
         {
             var result = new ConcurrentDictionary<LicenseNameAndVersion, LicenseValidationResult>();
             await foreach (ReferencedPackageWithContext info in packages)
@@ -45,7 +46,7 @@ public async Task<IEnumerable<LicenseValidationResult>> Validate(
                 }
                 else if (info.PackageInfo.LicenseUrl != null)
                 {
-                    await ValidateLicenseByUrl(info.PackageInfo, info.Context, result);
+                    await ValidateLicenseByUrl(info.PackageInfo, info.Context, result, token);
                 }
                 else
                 {
@@ -150,14 +151,20 @@ private void ValidateLicenseByMetadata(IPackageMetadata info,
 
         private async Task ValidateLicenseByUrl(IPackageMetadata info,
             string context,
-            ConcurrentDictionary<LicenseNameAndVersion, LicenseValidationResult> result)
+            ConcurrentDictionary<LicenseNameAndVersion, LicenseValidationResult> result,
+            CancellationToken token)
         {
             if (info.LicenseUrl!.IsAbsoluteUri)
             {
                 try
                 {
                     await _fileDownloader.DownloadFile(info.LicenseUrl,
-                        $"{info.Identity.Id}__{info.Identity.Version}.html");
+                        $"{info.Identity.Id}__{info.Identity.Version}.html",
+                        token);
+                }
+                catch (OperationCanceledException)
+                {
+                    // swallow cancellation
                 }
                 catch (Exception e)
                 {
@@ -207,7 +214,7 @@ private bool IsLicenseValid(string licenseId)
                 return true;
             }
 
-            return _allowedLicenses.Any(l => l.Equals(licenseId));
+            return _allowedLicenses.Any(allowedLicense => allowedLicense.Equals(licenseId));
         }
 
         private string GetLicenseNotAllowedMessage(string license)

src/NuGetUtility/Program.cs

@@ -124,7 +124,7 @@ private async Task<int> OnExecuteAsync(CancellationToken cancellationToken)
             });
             IAsyncEnumerable<ReferencedPackageWithContext> downloadedLicenseInformation =
                 packagesForProject.SelectMany(p => GetPackageInfos(p, overridePackageInformation, cancellationToken));
-            var results = (await validator.Validate(downloadedLicenseInformation)).ToList();
+            var results = (await validator.Validate(downloadedLicenseInformation, cancellationToken)).ToList();
 
             if (projectReaderExceptions.Any())
             {

src/NuGetUtility/Wrapper/HttpClientWrapper/FileDownloader.cs

@@ -1,22 +1,48 @@
-namespace NuGetUtility.Wrapper.HttpClientWrapper
+using System;
+
+namespace NuGetUtility.Wrapper.HttpClientWrapper
 {
     public class FileDownloader : IFileDownloader
     {
+        private readonly SemaphoreSlim _parallelDownloadLimiter = new SemaphoreSlim(10, 10);
         private readonly HttpClient _client;
         private readonly string _downloadDirectory;
+        private const int EXPONENTIAL_BACKOFF_WAIT_TIME_MILLISECONDS = 200;
+        private const int MAX_RETRIES = 5;
 
         public FileDownloader(HttpClient client, string downloadDirectory)
         {
             _client = client;
             _downloadDirectory = downloadDirectory;
         }
 
-        public async Task DownloadFile(Uri url, string fileName)
+        public async Task DownloadFile(Uri url, string fileName, CancellationToken token)
         {
-            await using FileStream file = File.OpenWrite(Path.Combine(_downloadDirectory, fileName));
-            await using Stream downloadStream = await _client.GetStreamAsync(url);
+            await _parallelDownloadLimiter.WaitAsync(token);
+            try
+            {
+                for (int i = 0; i < MAX_RETRIES; i++)
+                {
+                    await using FileStream file = File.OpenWrite(Path.Combine(_downloadDirectory, fileName));
+                    var request = new HttpRequestMessage(HttpMethod.Get, url);
+
+                    HttpResponseMessage response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
+                    response.EnsureSuccessStatusCode();
+                    if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
+                    {
+                        await Task.Delay((int)Math.Pow(EXPONENTIAL_BACKOFF_WAIT_TIME_MILLISECONDS, i + 1), token);
+                        continue;
+                    }
+                    using Stream downloadStream = await response.Content.ReadAsStreamAsync(token);
 
-            await downloadStream.CopyToAsync(file);
+                    await downloadStream.CopyToAsync(file, token);
+                    return;
+                }
+            }
+            finally
+            {
+                _parallelDownloadLimiter.Release();
+            }
         }
     }
 }

src/NuGetUtility/Wrapper/HttpClientWrapper/IFileDownloader.cs

@@ -2,6 +2,6 @@
 {
     public interface IFileDownloader
     {
-        public Task DownloadFile(Uri url, string fileName);
+        public Task DownloadFile(Uri url, string fileName, CancellationToken token);
     }
 }

src/NuGetUtility/Wrapper/HttpClientWrapper/NopFileDownloader.cs

@@ -2,7 +2,7 @@
 {
     public class NopFileDownloader : IFileDownloader
     {
-        public Task DownloadFile(Uri url, string fileName)
+        public Task DownloadFile(Uri url, string fileName, CancellationToken token)
         {
             return Task.CompletedTask;
         }