Merge branch 'resolve-deps-via-index'

source/Reloaded.Mod.Launcher.Lib/Commands/Mod/PublishModCommand.cs

@@ -21,6 +21,9 @@ public PublishModCommand(PathTuple<ModConfig>? modTuple)
     /// <inheritdoc />
     public void Execute(object? parameter)
     {
+        if (Update.CheckMissingDependencies().AllAvailable)
+            Task.Run(async () => await DependencyMetadataWriterFactory.ExecuteAllAsync(IoC.Get<ModConfigService>())).Wait();
+
         if (!NuGetVersion.TryParse(_modTuple!.Config.ModVersion, out var version))
         {
             Actions.DisplayMessagebox(Resources.ErrorInvalidModConfigTitle.Get(), Resources.ErrorInvalidModConfigDescription.Get());

source/Reloaded.Mod.Launcher.Lib/Setup.cs

@@ -125,7 +125,7 @@ public static bool TryGetIncompatibleMods(PathTuple<ApplicationConfig> applicati
     /// Quickly checks for any missing dependencies amongst available mods
     /// and opens a menu allowing to download if mods are unavailable.
     /// </summary>
-    /// <returns></returns>
+    /// <returns>True if there are missing deps, else false.</returns>
     public static async Task CheckForMissingModDependenciesAsync()
     {
         var deps = Update.CheckMissingDependencies();
@@ -310,8 +310,12 @@ private static void SetLoaderPaths(LoaderConfig config, string launcherDirectory
     /// </summary>
     private static async Task CheckForUpdatesAsync()
     {
-        await CheckForMissingModDependenciesAsync(); // in case DependencyMetadataWriterFactory deletes one after unsuccessful pull from 1 click
-        await DependencyMetadataWriterFactory.ExecuteAllAsync(IoC.Get<ModConfigService>());
+        // The action below is destructive.
+        // It may remove update metadata for missing dependencies.
+        // Don't run this unless we have all the mods.
+        if (Update.CheckMissingDependencies().AllAvailable)
+            await DependencyMetadataWriterFactory.ExecuteAllAsync(IoC.Get<ModConfigService>());
+
         await Update.CheckForLoaderUpdatesAsync();
         await Task.Run(Update.CheckForModUpdatesAsync);
         await CheckForMissingModDependenciesAsync();

source/Reloaded.Mod.Launcher.Lib/Update.cs

@@ -1,4 +1,5 @@
 using Reloaded.Mod.Loader.Update.Providers.GitHub;
+using Reloaded.Mod.Loader.Update.Providers.Index;
 using Constants = Reloaded.Mod.Launcher.Lib.Misc.Constants;
 using Version = Reloaded.Mod.Launcher.Lib.Utility.Version;
 
@@ -257,7 +258,9 @@ public static async Task<ModDependencyResolveResult> GetMissingDependenciesToDow
     {
         // Get missing dependencies for this update loop.
         var missingDeps = CheckMissingDependencies();
-
+        if (missingDeps.AllAvailable)
+            return ModDependencyResolveResult.Combine(Enumerable.Empty<ModDependencyResolveResult>());
+
         // Get Dependencies
         var resolver = DependencyResolverFactory.GetInstance(IoC.Get<AggregateNugetRepository>());
 
@@ -269,7 +272,18 @@ public static async Task<ModDependencyResolveResult> GetMissingDependenciesToDow
         await Task.WhenAll(results);
 
         // Merge Results
-        return ModDependencyResolveResult.Combine(results.Select(x => x.Result));;
+        var result = ModDependencyResolveResult.Combine(results.Select(x => x.Result));;
+        if (result.NotFoundDependencies.Count <= 0)
+            return result;
+
+        // Fallback to using Index Resolver if we couldn't find the package otherwise.
+        var indexResolver = new IndexDependencyResolver();
+        var indexResults = new List<ModDependencyResolveResult>();
+        foreach (var notFound in result.NotFoundDependencies)
+            indexResults.Add(await indexResolver.ResolveAsync(notFound, null, token));
+
+        indexResults.Add(result);
+        return ModDependencyResolveResult.Combine(indexResults);
     }
 
     /// <summary>

source/Reloaded.Mod.Loader.Tests/Index/BuildIndex.cs

@@ -42,6 +42,26 @@ public async Task BuildCombinedIndex()
         Assert.True(index.TryGetGameBananaSourcePath(heroesGameId, out _));
         Assert.True(Directory.Exists(index.BaseUrl.LocalPath));
     }
+
+    [Fact]
+    public async Task BuildAllPackages()
+    {
+        const string outputFolder = "BuildAllPackagesTest";
+        const int heroesGameId = 6061;
+
+        // Arrange
+        var builder = new IndexBuilder();
+        builder.Sources.Add(new IndexSourceEntry(TestNuGetFeedUrl));
+        builder.Sources.Add(new IndexSourceEntry(heroesGameId));
+
+        // Act
+        var index = await builder.BuildAsync(outputFolder);
+
+        // Assert
+        var api = new IndexApi(index.BaseUrl.ToString());
+        var packages = await api.GetAllPackagesAsync();
+        Assert.True(packages.Packages.Count > 0);
+    }
 
     [Fact]
     public async Task BuildGbIndex()

source/Reloaded.Mod.Loader.Tests/Update/Providers/Index/IndexDependencyResolverTests.cs

@@ -0,0 +1,46 @@
+using Reloaded.Mod.Loader.Update.Providers.Index;
+
+namespace Reloaded.Mod.Loader.Tests.Update.Providers.Index;
+
+public class IndexDependencyResolverTests
+{
+    [Fact]
+    public async Task ResolveAsync_WithExistingPackage_ReturnsPackages()
+    {
+        // Act
+        var resolver = new IndexDependencyResolver();
+        var result = await resolver.ResolveAsync("reloaded.universal.redirector");
+
+        // Assert
+        Assert.NotEmpty(result.FoundDependencies);
+        Assert.Empty(result.NotFoundDependencies);
+    }
+
+    [Fact]
+    public async Task ResolveAsync_WithExistingPackage_CanDownloadPackage()
+    {
+        // Act
+        using var outputDirectory = new TemporaryFolderAllocation();
+        var resolver = new IndexDependencyResolver();
+        var result = await resolver.ResolveAsync("reloaded.universal.redirector");
+        var downloadedPackagePath = await result.FoundDependencies[0].DownloadAsync(outputDirectory.FolderPath, null);
+
+        // Assert
+        Assert.True(Directory.Exists(downloadedPackagePath));
+        Assert.True(File.Exists(Path.Combine(downloadedPackagePath, ModConfig.ConfigFileName)));
+    }
+
+    [Fact]
+    public async Task ResolveAsync_WithNoPackage_ReturnsMissing()
+    {
+        // Arrange
+        var resolver = new IndexDependencyResolver();
+
+        // Act
+        var result = await resolver.ResolveAsync("this.package.does.not.exist");
+
+        // Assert
+        Assert.NotEmpty(result.NotFoundDependencies);
+        Assert.Empty(result.FoundDependencies);
+    }
+}

source/Reloaded.Mod.Loader.Update/DependencyResolverFactory.cs

@@ -1,3 +1,5 @@
+using Reloaded.Mod.Loader.Update.Providers.Index;
+
 namespace Reloaded.Mod.Loader.Update;
 
 /// <summary>
@@ -10,11 +12,10 @@ public static class DependencyResolverFactory
     /// </summary>
     public static IDependencyResolver GetInstance(AggregateNugetRepository repository)
     {
-        return new AggregateDependencyResolver(new IDependencyResolver[]
-        {
+        return new AggregateDependencyResolver([
             new NuGetDependencyResolver(repository),
             new GameBananaDependencyResolver(),
             new GitHubDependencyResolver()
-        });
+        ]);
     }
 }

source/Reloaded.Mod.Loader.Update/Index/IndexApi.cs

@@ -36,6 +36,24 @@ public IndexApi(string indexUrl = "https://reloaded-project.github.io/Reloaded-I
         index.BaseUrl = IndexUrl;
         return index;
     }
+
+    /// <summary>
+    /// Returns all packages from the index.
+    /// </summary>
+    public async Task<PackageList> GetAllPackagesAsync()
+    {
+        var uri = new Uri(IndexUrl, Routes.AllPackages);
+        return await Web.DownloadAndDeserialize<PackageList>(uri);
+    }
+
+    /// <summary>
+    /// Returns all packages from the index.
+    /// </summary>
+    public async Task<PackageList> GetAllDependenciesAsync()
+    {
+        var uri = new Uri(IndexUrl, Routes.AllDependencies);
+        return await Web.DownloadAndDeserialize<PackageList>(uri);
+    }
 }
 
 /// <summary>

source/Reloaded.Mod.Loader.Update/Index/IndexBuilder.cs

@@ -22,7 +22,7 @@ public class IndexBuilder
         Directory.CreateDirectory(outputFolder);
         var index     = new Structures.Index();
         index.BaseUrl = new Uri($"{outputFolder}/");
-        return await UpdateAsync(index);
+        return await UpdateAsync(index, writeToFile);
     }
 
     /// <summary>
@@ -86,7 +86,13 @@ public Structures.Index RemoveNotInBuilder(Structures.Index index)
 
         await Task.WhenAll(tasks);
         if (writeToFile)
+        {
             await WriteToDiskAsync(index);
+            var allPackages = await index.GetPackagesFromAllSourcesAsync();
+            await WriteToDiskAsync(index.BaseUrl, allPackages, Routes.AllPackages);
+            allPackages.RemoveNonDependencyInfo();
+            await WriteToDiskAsync(index.BaseUrl, allPackages, Routes.AllDependencies);
+        }
 
         index.BaseUrl = new Uri(outputFolder, UriKind.Absolute);
         return index;
@@ -102,16 +108,28 @@ public async Task WriteToDiskAsync(Structures.Index index)
         await File.WriteAllBytesAsync(Path.Combine(index.BaseUrl.LocalPath, Routes.Index), compressedIndex);
     }
 
+    /// <summary>
+    /// Writes an existing package list to disk, in a specified folder.
+    /// </summary>
+    /// <param name="list">The list containing all packages.</param>
+    /// <param name="baseUrl">The 'base URL' where the Index is contained.</param>
+    /// <param name="route">The route where this package list goes.</param>
+    public async Task WriteToDiskAsync(Uri baseUrl, PackageList list, string route)
+    {
+        var compressedPackageList = Compression.Compress(JsonSerializer.SerializeToUtf8Bytes(list, Serializer.Options));
+        await File.WriteAllBytesAsync(Path.Combine(baseUrl.LocalPath, route), compressedPackageList);
+    }
+
     private async Task BuildNuGetSourceAsync(Structures.Index index, IndexSourceEntry indexSourceEntry,
         string outputFolder)
     {
         // Number of items to grab at once.
-        const int Take = 500;
+        const int take = 500;
 
         var provider = new NuGetPackageProvider(NugetRepository.FromSourceUrl(indexSourceEntry.NuGetUrl!), null, false);
 
         var packagesList = PackageList.Create();
-        await SearchForAllResults(Take, provider, packagesList);
+        await SearchForAllResults(take, provider, packagesList);
 
         var relativePath = Routes.Build.GetNuGetPackageListPath(indexSourceEntry.NuGetUrl!);
         var path = Path.Combine(outputFolder, relativePath);
@@ -121,7 +139,8 @@ private async Task BuildNuGetSourceAsync(Structures.Index index, IndexSourceEntr
         index.Sources[Routes.Source.GetNuGetIndexKey(indexSourceEntry.NuGetUrl!)] = relativePath;
     }
 
-    private async Task BuildGameBananaSourceAsync(Structures.Index index, IndexSourceEntry indexSourceEntry, string outputFolder)
+    private async Task BuildGameBananaSourceAsync(Structures.Index index, IndexSourceEntry indexSourceEntry,
+        string outputFolder)
     {
         // Max for GameBanana
         const int take = 50;

source/Reloaded.Mod.Loader.Update/Index/Routes.cs

@@ -19,6 +19,19 @@ public static class Routes
     /// Link to the index containing the mappings
     /// </summary>
     public const string Index = $"Index{FileExtension}{CompressionExtension}";
+
+    /// <summary>
+    /// Link to the index containing all packages
+    /// </summary>
+    public const string AllPackages = $"AllPackages{FileExtension}{CompressionExtension}";
+
+    /// <summary>
+    /// Link to the index containing all dependencies
+    /// </summary>
+    /// <remarks>
+    ///     This is same as <see cref="AllPackages"/> but with information omitted to reduce bandwidth.
+    /// </remarks>
+    public const string AllDependencies = $"AllDependencies{FileExtension}{CompressionExtension}";
 
     /// <summary>
     /// Contains the code to determine locations of output files

source/Reloaded.Mod.Loader.Update/Index/Structures/Index.cs

@@ -17,6 +17,26 @@ public class Index
     [JsonIgnore]
     public Uri BaseUrl { get; internal set; } = null!;
 
+
+    /// <summary>
+    /// [Slow if over network/internet !!]
+    /// Retrieves all the packages from all the sources.
+    /// </summary>
+    public async Task<PackageList> GetPackagesFromAllSourcesAsync()
+    {
+        var result = new List<Package>();
+        foreach (var source in Sources)
+        {
+            var fromSource = await Web.DownloadAndDeserialize<PackageList>(new Uri(BaseUrl, source.Value));
+            result.AddRange(fromSource.Packages);
+        }
+
+        return new PackageList
+        {
+            Packages = result
+        };
+    }
+
     /// <summary>
     /// Tries to get the package list for a given NuGet URL.
     /// </summary>

source/Reloaded.Mod.Loader.Update/Index/Structures/Package.cs

@@ -88,4 +88,23 @@ public async Task<string> DownloadAsync(string packageFolder, IProgress<double>?
         this.Adapt(webPackage);
         return await webPackage.DownloadAsync(packageFolder, progress, token);
     }
+
+    /// <summary>
+    /// Removes all information that is not needed in dependency resolution.
+    /// </summary>
+    public void RemoveNonDependencyInfo()
+    {
+        Authors = null;
+        Submitter = null;
+        Description = null;
+        MarkdownReadme = null;
+        Images = null;
+        ProjectUri = null;
+        LikeCount = null;
+        ViewCount = null;
+        DownloadCount = null;
+        Published = null;
+        Changelog = null;
+        Tags = null;
+    }
 }

source/Reloaded.Mod.Loader.Update/Index/Structures/PackageList.cs

@@ -22,4 +22,15 @@ public static PackageList Create()
             Packages = new List<Package>()
         };
     }
+
+    /// <summary>
+    /// Removes the info that is not needed for dependency resolution.
+    /// </summary>
+    public void RemoveNonDependencyInfo()
+    {
+        foreach (var package in Packages)
+        {
+            package.RemoveNonDependencyInfo();
+        }
+    }
 }

source/Reloaded.Mod.Loader.Update/Interfaces/IDependencyResolver.cs

@@ -61,6 +61,10 @@ public static ModDependencyResolveResult Combine(IEnumerable<ModDependencyResolv
                 returnValue.NotFoundDependencies.Add(notFound);
         }
 
+        // Remove dependencies that were found from the notFound set.
+        foreach (var found in idToNewestVersion.Keys)
+            returnValue.NotFoundDependencies.Remove(found);
+
         returnValue.FoundDependencies.AddRange(idToNewestVersion.Values);
         return returnValue;
     }