From 4b7030bf74b0e1f13857b1b5eba67575ef2979e7 Mon Sep 17 00:00:00 2001 From: shartte Date: Fri, 19 Jul 2024 16:32:54 +0200 Subject: [PATCH] Enhanced asset download support (#25) - Add support for copying Assets from an existing Launcher - Add support for writing the Neoform asset JSON reference format - Add support for specifying the asset download directory --- README.md | 15 +- build.gradle | 6 +- .../CreateLibrariesOptionsFileAction.java | 2 +- .../DownloadFromVersionManifestAction.java | 2 +- .../runtime/actions/PatchActionFactory.java | 1 - .../neoform/runtime/artifacts/Artifact.java | 7 + .../runtime/artifacts/ArtifactManager.java | 14 +- .../neoform/runtime/cache/CacheManager.java | 4 +- .../runtime/cache/LauncherInstallations.java | 94 ++++- .../runtime/cli/DownloadAssetsCommand.java | 235 ++++------- .../neoform/runtime/cli/LockManager.java | 1 - .../neoforged/neoform/runtime/cli/Main.java | 8 +- .../downloads/AssetDownloadResult.java | 40 ++ .../runtime/downloads/AssetDownloader.java | 174 ++++++++ .../runtime/downloads/DownloadManager.java | 25 +- .../downloads/DownloadsFailedException.java | 15 + .../runtime/downloads/ParallelDownloader.java | 149 +++++++ .../runtime/engine/NeoFormInterpolator.java | 4 - .../runtime/manifests/AssetObject.java | 3 + .../neoform/runtime/utils/FileUtil.java | 12 +- .../neoform/runtime/utils/HashingUtil.java | 10 +- .../downloads/AssetDownloadResultTest.java | 59 +++ .../downloads/AssetDownloaderTest.java | 379 ++++++++++++++++++ .../downloads/DownloadManagerTest.java | 239 +++++++++++ 24 files changed, 1281 insertions(+), 217 deletions(-) create mode 100644 src/main/java/net/neoforged/neoform/runtime/downloads/AssetDownloadResult.java create mode 100644 src/main/java/net/neoforged/neoform/runtime/downloads/AssetDownloader.java create mode 100644 src/main/java/net/neoforged/neoform/runtime/downloads/DownloadsFailedException.java create mode 100644 src/main/java/net/neoforged/neoform/runtime/downloads/ParallelDownloader.java create mode 100644 src/test/java/net/neoforged/neoform/runtime/downloads/AssetDownloadResultTest.java create mode 100644 src/test/java/net/neoforged/neoform/runtime/downloads/AssetDownloaderTest.java create mode 100644 src/test/java/net/neoforged/neoform/runtime/downloads/DownloadManagerTest.java diff --git a/README.md b/README.md index 45747d1..55c3699 100644 --- a/README.md +++ b/README.md @@ -69,13 +69,13 @@ NFRT helps with this by downloading the assets required to run a particular vers ``` # Download Assets for a specific version of Minecraft -nfrt download-assets --minecraft-version 1.20.6 --output-properties-to assets.properties +nfrt download-assets --minecraft-version 1.20.6 --write-properties assets.properties # Download Assets for the Minecraft version used by the given NeoForm version -nfrt download-assets --neoform net.neoforged:neoform:1.20.6-20240429.153634@zip --output-properties-to assets.properties +nfrt download-assets --neoform net.neoforged:neoform:1.20.6-20240429.153634@zip --write-properties assets.properties # Download Assets for the Minecraft version used by the given NeoForge version -nfrt download-assets --neoforge net.neoforged:neoforge:20.6.72-beta:userdev --output-properties-to assets.properties +nfrt download-assets --neoforge net.neoforged:neoforge:20.6.72-beta:userdev --write-properties assets.properties # In all three cases, a properties file will be written to assets.properties containing the following, # which can be used to pass the required command line arguments for starting a Minecraft client. @@ -86,6 +86,15 @@ assets_root=...path to assets... While it may seem odd that NFRT supports passing NeoForm or NeoForge versions to this command, this is in service of potential Gradle plugins never having to actually read and parse the NeoForm configuration file. +| Option | Description | +|--------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--asets-dir` | Where to store the downloaded assets. Optional. Defaults to `/assets`, or a detected Launcher installation. | +| `--no-copy-launcher-assets` | Disables copying of local Minecraft Launcher assets, if using the asset root directly is disabled. | +| `--no-use-launcher-asset-root` | Disables using a detected Minecraft Launcher installation directly to store the required assets. | +| `--concurrent-downloads` | Limits the maximum number of concurrent downloads. Default is 25. | +| `--write-properties` | Writes a property file to the given path that contains the asset index id (`asset_index`) and asset root path (`assets_root`) suitable for passing to Minecraft. | +| `--write-json` | Writes a JSON file to the given path that contains the asset index id (`asset_index`) and asset root path (`assets`) suitable for passing to Minecraft via a Neoform entrypoint. | + ## Common Options These options affect all NFRT subcommands. diff --git a/build.gradle b/build.gradle index c530621..f2020fb 100644 --- a/build.gradle +++ b/build.gradle @@ -101,8 +101,12 @@ dependencies { annotationProcessor 'info.picocli:picocli-codegen:4.7.6' testImplementation platform('org.junit:junit-bom:5.10.0') + testImplementation platform('org.mockito:mockito-bom:5.12.0') + testImplementation platform('org.assertj:assertj-bom:3.26.3') testImplementation 'org.junit.jupiter:junit-jupiter' testImplementation 'org.junit.jupiter:junit-jupiter-params' + testImplementation 'org.mockito:mockito-junit-jupiter' + testImplementation 'org.assertj:assertj-core' } // Add dependencies to external tools @@ -203,7 +207,7 @@ idea { } "Download assets 1.20.6"(Application) { mainClass = mainClassName - programParameters = "download-assets --neoforge net.neoforged:neoforge:20.6.72-beta:userdev --output-properties-to build/assets.properties" + programParameters = "download-assets --neoforge net.neoforged:neoforge:20.6.72-beta:userdev --write-properties build/assets.properties --write-json build/assets.json" moduleRef(project, sourceSets.main) } } diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/CreateLibrariesOptionsFileAction.java b/src/main/java/net/neoforged/neoform/runtime/actions/CreateLibrariesOptionsFileAction.java index 6e35f05..aca7856 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/CreateLibrariesOptionsFileAction.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/CreateLibrariesOptionsFileAction.java @@ -2,8 +2,8 @@ import net.neoforged.neoform.runtime.cache.CacheKeyBuilder; import net.neoforged.neoform.runtime.engine.ProcessingEnvironment; -import net.neoforged.neoform.runtime.graph.ResultRepresentation; import net.neoforged.neoform.runtime.graph.ExecutionNodeAction; +import net.neoforged.neoform.runtime.graph.ResultRepresentation; import java.io.IOException; import java.nio.file.Files; diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/DownloadFromVersionManifestAction.java b/src/main/java/net/neoforged/neoform/runtime/actions/DownloadFromVersionManifestAction.java index 6205232..3fee65a 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/DownloadFromVersionManifestAction.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/DownloadFromVersionManifestAction.java @@ -1,7 +1,7 @@ package net.neoforged.neoform.runtime.actions; -import net.neoforged.neoform.runtime.cache.CacheKeyBuilder; import net.neoforged.neoform.runtime.artifacts.ArtifactManager; +import net.neoforged.neoform.runtime.cache.CacheKeyBuilder; import net.neoforged.neoform.runtime.engine.ProcessingEnvironment; import net.neoforged.neoform.runtime.graph.ResultRepresentation; diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/PatchActionFactory.java b/src/main/java/net/neoforged/neoform/runtime/actions/PatchActionFactory.java index 6f1896e..4bc1178 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/PatchActionFactory.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/PatchActionFactory.java @@ -3,7 +3,6 @@ import net.neoforged.neoform.runtime.graph.ExecutionNodeBuilder; import net.neoforged.neoform.runtime.graph.NodeOutput; import net.neoforged.neoform.runtime.graph.NodeOutputType; -import net.neoforged.neoform.runtime.utils.MavenCoordinate; import net.neoforged.neoform.runtime.utils.ToolCoordinate; import java.nio.file.Path; diff --git a/src/main/java/net/neoforged/neoform/runtime/artifacts/Artifact.java b/src/main/java/net/neoforged/neoform/runtime/artifacts/Artifact.java index 8564a60..ababcfa 100644 --- a/src/main/java/net/neoforged/neoform/runtime/artifacts/Artifact.java +++ b/src/main/java/net/neoforged/neoform/runtime/artifacts/Artifact.java @@ -1,6 +1,13 @@ package net.neoforged.neoform.runtime.artifacts; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; public record Artifact(Path path, long lastModified, long size) { + public static Artifact ofPath(Path path) throws IOException { + var attributes = Files.readAttributes(path, BasicFileAttributes.class); + return new Artifact(path, attributes.lastModifiedTime().toMillis(), attributes.size()); + } } diff --git a/src/main/java/net/neoforged/neoform/runtime/artifacts/ArtifactManager.java b/src/main/java/net/neoforged/neoform/runtime/artifacts/ArtifactManager.java index bc784c1..9265331 100644 --- a/src/main/java/net/neoforged/neoform/runtime/artifacts/ArtifactManager.java +++ b/src/main/java/net/neoforged/neoform/runtime/artifacts/ArtifactManager.java @@ -82,7 +82,7 @@ public Artifact get(MinecraftLibrary library) throws IOException { // Ensure the file matches before using it var fileHash = HashingUtil.hashFile(localPath, artifact.checksumAlgorithm()); if (Objects.equals(fileHash, artifact.checksum())) { - return getArtifactFromPath(localPath); + return Artifact.ofPath(localPath); } } catch (IOException ignored) { // Ignore if it doesn't exist or is otherwise fails to be read @@ -175,7 +175,7 @@ public Artifact getVersionManifest(String minecraftVersion) throws IOException { for (var root : launcherInstallations.getInstallationRoots()) { var localPath = root.resolve("versions").resolve(minecraftVersion).resolve(minecraftVersion + ".json"); if (Files.isReadable(localPath)) { - return getArtifactFromPath(localPath); + return Artifact.ofPath(localPath); } } @@ -250,15 +250,7 @@ public void loadArtifactManifest(Path artifactManifestPath) throws IOException { } private Artifact getArtifactFromPath(String path) throws IOException { - return getArtifactFromPath(Paths.get(path)); - } - - private Artifact getArtifactFromPath(Path path) throws IOException { - if (!Files.isRegularFile(path)) { - throw new NoSuchFileException(path.toString()); - } - var attrView = Files.getFileAttributeView(path, BasicFileAttributeView.class).readAttributes(); - return new Artifact(path, attrView.lastModifiedTime().toMillis(), attrView.size()); + return Artifact.ofPath(Paths.get(path)); } @FunctionalInterface diff --git a/src/main/java/net/neoforged/neoform/runtime/cache/CacheManager.java b/src/main/java/net/neoforged/neoform/runtime/cache/CacheManager.java index 5b129f4..5592ab1 100644 --- a/src/main/java/net/neoforged/neoform/runtime/cache/CacheManager.java +++ b/src/main/java/net/neoforged/neoform/runtime/cache/CacheManager.java @@ -73,12 +73,12 @@ public class CacheManager implements AutoCloseable { private boolean analyzeMisses; private boolean verbose; - public CacheManager(Path homeDir, Path workspacesDir) throws IOException { + public CacheManager(Path homeDir, @Nullable Path assetsDir, Path workspacesDir) throws IOException { this.homeDir = homeDir; Files.createDirectories(homeDir); this.artifactCacheDir = homeDir.resolve("artifacts"); this.intermediateResultsDir = homeDir.resolve("intermediate_results"); - this.assetsDir = homeDir.resolve("assets"); + this.assetsDir = Objects.requireNonNullElse(assetsDir, homeDir.resolve("assets")); this.workspacesDir = workspacesDir; } diff --git a/src/main/java/net/neoforged/neoform/runtime/cache/LauncherInstallations.java b/src/main/java/net/neoforged/neoform/runtime/cache/LauncherInstallations.java index 000d8aa..bd1bacc 100644 --- a/src/main/java/net/neoforged/neoform/runtime/cache/LauncherInstallations.java +++ b/src/main/java/net/neoforged/neoform/runtime/cache/LauncherInstallations.java @@ -4,11 +4,16 @@ import net.neoforged.neoform.runtime.utils.Logger; import org.jetbrains.annotations.Nullable; +import java.io.IOException; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -39,19 +44,23 @@ public class LauncherInstallations { }; private static final Pattern PLACEHOLDER_PATTERN = Pattern.compile("\\$\\{([^}]+)}"); + // Sort launcher directories in descending order by the number of asset indices they contain + private static final Comparator ASSET_INDEX_COUNT_DESCENDING = Comparator.comparingInt(d -> d.assetIndices.size()).reversed(); + private final List launcherDirectories = new ArrayList<>(); - /** If true, the scan was already performed and launcherDirectories is up-to-date */ + /** If true, the scan was already performed and launcherDirectories is up to date */ private boolean scanned; private boolean verbose; - public LauncherInstallations(List additionalLauncherDirs) { + public LauncherInstallations(List additionalLauncherDirs) throws IOException { for (var dir : additionalLauncherDirs) { var launcherDir = analyzeLauncherDirectory(dir); - if (launcherDir != null) { - launcherDirectories.add(launcherDir); + if (launcherDir == null) { + throw new NoSuchFileException(dir.toString()); } + launcherDirectories.add(launcherDir); } } @@ -70,6 +79,51 @@ public List getInstallationRoots() { return launcherDirectories.stream().map(LauncherDirectory::directory).toList(); } + /** + * Try finding an asset root in any of the known launcher installations that already has + * an asset index with the given id. + * + * @return Null if no such launcher installation could be found. + */ + @Nullable + public Path getAssetDirectoryForIndex(String assetIndexId) { + scanIfNecessary(); + + var haveIndex = new ArrayList(); + for (var launcherDirectory : launcherDirectories) { + if (launcherDirectory.assetIndices.contains(assetIndexId)) { + haveIndex.add(launcherDirectory); + } + } + + // Sort by count of other indices descending + haveIndex.sort(ASSET_INDEX_COUNT_DESCENDING); + + if (!haveIndex.isEmpty()) { + return haveIndex.getFirst().assetDirectory(); + } + + return null; + } + + /** + * Returns the list of asset directories found among the known launcher installations. + */ + public List getAssetRoots() { + scanIfNecessary(); + + var launcherDirsWithAssets = new ArrayList(); + for (var launcherDirectory : launcherDirectories) { + if (launcherDirectory.assetDirectory() != null) { + launcherDirsWithAssets.add(launcherDirectory); + } + } + + launcherDirsWithAssets.sort(ASSET_INDEX_COUNT_DESCENDING); + + return launcherDirsWithAssets.stream().map(LauncherDirectory::assetDirectory).toList(); + } + private void scanIfNecessary() { if (scanned) { return; @@ -112,8 +166,15 @@ private void scanIfNecessary() { if (verbose) { LOG.println("Launcher directories found:"); for (var launcherDirectory : launcherDirectories) { + String details; + if (launcherDirectory.assetDirectory == null || launcherDirectory.assetIndices().isEmpty()) { + details = "no assets"; + } else { + details = "asset indices: " + String.join(" ", launcherDirectory.assetIndices()); + } + LOG.println(AnsiColor.MUTED + " " + launcherDirectory.directory - + AnsiColor.RESET); + + " (" + details + ")" + AnsiColor.RESET); } } } @@ -148,7 +209,7 @@ private String resolvePlaceholders(String candidate) { } @Nullable - private LauncherDirectory analyzeLauncherDirectory(Path installDir) { + private LauncherDirectory analyzeLauncherDirectory(Path installDir) throws IOException { if (!Files.isDirectory(installDir)) { if (verbose) { LOG.println(AnsiColor.MUTED + " Not found: " + installDir + AnsiColor.RESET); @@ -156,9 +217,26 @@ private LauncherDirectory analyzeLauncherDirectory(Path installDir) { return null; } - return new LauncherDirectory(installDir); + var assetIndices = new HashSet(); + var assetRoot = installDir.resolve("assets"); + var assetIndicesDir = assetRoot.resolve("indexes"); + var assetObjectsDir = assetRoot.resolve("objects"); + if (Files.isDirectory(assetIndicesDir) && Files.isDirectory(assetObjectsDir)) { + // Count the number of asset indices present to judge how viable this directory is + try (var stream = Files.list(assetIndicesDir)) { + stream.map(f -> f.getFileName().toString()) + .filter(f -> f.endsWith(".json")) + .map(f -> f.substring(0, f.length() - 5)) + .forEach(assetIndices::add); + } + } + if (assetIndices.isEmpty()) { + assetRoot = null; // Do not use an asset root, when it is empty + } + + return new LauncherDirectory(installDir, assetRoot, assetIndices); } - private record LauncherDirectory(Path directory) { + private record LauncherDirectory(Path directory, @Nullable Path assetDirectory, Set assetIndices) { } } diff --git a/src/main/java/net/neoforged/neoform/runtime/cli/DownloadAssetsCommand.java b/src/main/java/net/neoforged/neoform/runtime/cli/DownloadAssetsCommand.java index 0c00de2..8ecbf63 100644 --- a/src/main/java/net/neoforged/neoform/runtime/cli/DownloadAssetsCommand.java +++ b/src/main/java/net/neoforged/neoform/runtime/cli/DownloadAssetsCommand.java @@ -1,43 +1,27 @@ package net.neoforged.neoform.runtime.cli; import net.neoforged.neoform.runtime.artifacts.ArtifactManager; -import net.neoforged.neoform.runtime.cache.CacheManager; import net.neoforged.neoform.runtime.config.neoforge.NeoForgeConfig; import net.neoforged.neoform.runtime.config.neoform.NeoFormConfig; +import net.neoforged.neoform.runtime.downloads.AssetDownloadResult; +import net.neoforged.neoform.runtime.downloads.AssetDownloader; import net.neoforged.neoform.runtime.downloads.DownloadManager; -import net.neoforged.neoform.runtime.downloads.DownloadSpec; -import net.neoforged.neoform.runtime.manifests.AssetIndex; -import net.neoforged.neoform.runtime.manifests.AssetObject; -import net.neoforged.neoform.runtime.manifests.MinecraftVersionManifest; -import net.neoforged.neoform.runtime.utils.Logger; +import net.neoforged.neoform.runtime.downloads.DownloadsFailedException; import net.neoforged.neoform.runtime.utils.MavenCoordinate; -import net.neoforged.neoform.runtime.utils.StringUtil; -import org.jetbrains.annotations.Nullable; import picocli.CommandLine; -import java.io.BufferedOutputStream; import java.io.IOException; import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HexFormat; -import java.util.Properties; +import java.nio.file.Path; import java.util.concurrent.Callable; -import java.util.concurrent.Executors; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.jar.JarFile; import java.util.zip.ZipFile; +/** + * @see AssetDownloader + */ @CommandLine.Command(name = "download-assets", description = "Download the client assets used to run a particular game version") public class DownloadAssetsCommand implements Callable { - private static final Logger LOG = Logger.create(); - - private static final ThreadFactory DOWNLOAD_THREAD_FACTORY = r -> Thread.ofVirtual().name("download-asset", 1).unstarted(r); - @CommandLine.ParentCommand Main commonOptions; @@ -47,14 +31,38 @@ public class DownloadAssetsCommand implements Callable { @CommandLine.Option(names = "--asset-repository") public URI assetRepository = URI.create("https://resources.download.minecraft.net/"); + @CommandLine.Option( + names = "--copy-launcher-assets", + description = "Try to find the Minecraft Launcher in common locations and copy its assets", + negatable = true, + fallbackValue = "true" + ) + public boolean copyLauncherAssets = true; + + @CommandLine.Option( + names = "--use-launcher-asset-root", + description = "Try to find an existing Minecraft Launcher asset root, and use it to store the requested assets", + negatable = true, + fallbackValue = "true" + ) + public boolean useLauncherAssetRoot = true; + @CommandLine.Option(names = "--concurrent-downloads") public int concurrentDownloads = 25; /** * Properties file that will receive the metadata of the asset index. */ - @CommandLine.Option(names = "--output-properties-to") - public String outputPropertiesPath; + @CommandLine.Option(names = "--write-properties") + public Path outputPropertiesPath; + + /** + * Write a JSON file as it is used by the Neoform Start.java file + * See: + * https://github.com/neoforged/NeoForm/blob/c2f5c5eda5eeca2e554c51872c28d0e68bc244bc/versions/release/1.21/inject/mcp/client/Start.java + */ + @CommandLine.Option(names = "--write-json") + public Path outputJsonPath; public static class Version { @CommandLine.Option(names = "--minecraft-version") @@ -65,6 +73,45 @@ public static class Version { String neoforgeArtifact; } + @Override + public Integer call() throws Exception { + try (var downloadManager = new DownloadManager(); + var cacheManager = commonOptions.createCacheManager(); + var lockManager = commonOptions.createLockManager()) { + + var launcherInstallations = commonOptions.createLauncherInstallations(); + var artifactManager = commonOptions.createArtifactManager(cacheManager, downloadManager, lockManager, launcherInstallations); + + var minecraftVersion = getMinecraftVersion(artifactManager); + + var downloader = new AssetDownloader(downloadManager, artifactManager, launcherInstallations, cacheManager); + AssetDownloadResult result; + try { + result = downloader.downloadAssets( + minecraftVersion, + assetRepository, + useLauncherAssetRoot, + copyLauncherAssets, + concurrentDownloads + ); + } catch (DownloadsFailedException e) { + System.err.println(e.getErrors().size() + " files failed to download"); + System.err.println("First error:"); + e.getErrors().getFirst().printStackTrace(); + return 1; + } + + if (outputPropertiesPath != null) { + result.writeAsProperties(outputPropertiesPath); + } + + if (outputJsonPath != null) { + result.writeAsJson(outputJsonPath); + } + return 0; + } + } + private String getMinecraftVersion(ArtifactManager artifactManager) throws IOException { if (version.minecraftVersion != null) { return version.minecraftVersion; @@ -88,142 +135,4 @@ private String getMinecraftVersion(ArtifactManager artifactManager) throws IOExc } } - @Override - public Integer call() throws Exception { - try (var downloadManager = new DownloadManager(); - var cacheManager = commonOptions.createCacheManager(); - var lockManager = commonOptions.createLockManager()) { - - var launcherInstallations = commonOptions.createLauncherInstallations(); - var artifactManager = commonOptions.createArtifactManager(cacheManager, downloadManager, lockManager, launcherInstallations); - - return downloadAssets(downloadManager, artifactManager, cacheManager); - } - } - - private int downloadAssets(DownloadManager downloadManager, - ArtifactManager artifactManager, - CacheManager cacheManager) throws IOException { - - var minecraftVersion = getMinecraftVersion(artifactManager); - - var versionManifest = MinecraftVersionManifest.from(artifactManager.getVersionManifest(minecraftVersion).path()); - var assetIndexReference = versionManifest.assetIndex(); - LOG.println("Downloading asset index " + assetIndexReference.id()); - - var assetRoot = cacheManager.getAssetsDir(); - var indexFolder = assetRoot.resolve("indexes"); - Files.createDirectories(indexFolder); - var objectsFolder = assetRoot.resolve("objects"); - Files.createDirectories(objectsFolder); - - var assetIndexPath = indexFolder.resolve(assetIndexReference.id() + ".json"); - downloadManager.download(assetIndexReference, assetIndexPath); - - // Pre-create all folders - for (var i = 0; i < 256; i++) { - var objectSubFolder = objectsFolder.resolve(HexFormat.of().toHexDigits(i, 2)); - Files.createDirectories(objectSubFolder); - } - - AtomicInteger downloadsDone = new AtomicInteger(); - AtomicLong bytesDownloaded = new AtomicLong(); - var errors = new ArrayList(); - var assetIndex = AssetIndex.from(assetIndexPath); - // The same object can be referenced multiple times - var objectsToDownload = assetIndex.objects().values().stream() - .distinct() - .filter(obj -> Files.notExists(objectsFolder.resolve(getObjectPath(obj)))) - .toList(); - - if (concurrentDownloads < 1) { - throw new IllegalStateException("Cannot set concurrent downloads to less than 1: " + concurrentDownloads); - } - - var semaphore = new Semaphore(concurrentDownloads); - try (var executor = Executors.newThreadPerTaskExecutor(DOWNLOAD_THREAD_FACTORY)) { - for (var object : objectsToDownload) { - var spec = new AssetDownloadSpec(object); - var objectHash = object.hash(); - String objectPath = objectHash.substring(0, 2) + "/" + objectHash; - var objectLocation = objectsFolder.resolve(objectPath); - executor.execute(() -> { - boolean hasAcquired = false; - try { - semaphore.acquire(); - hasAcquired = true; - if (downloadManager.download(spec, objectLocation, true)) { - bytesDownloaded.addAndGet(spec.size()); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (Exception e) { - synchronized (errors) { - errors.add(e); - } - } finally { - if (hasAcquired) { - semaphore.release(); - } - var finished = downloadsDone.incrementAndGet(); - if (finished % 100 == 0) { - LOG.println(finished + "/" + objectsToDownload.size() + " downloads"); - } - } - }); - } - } - LOG.println("Downloaded " + objectsToDownload.size() + " assets with a total size of " + StringUtil.formatBytes(bytesDownloaded.get())); - - if (!errors.isEmpty()) { - System.err.println(errors.size() + " files failed to download"); - System.err.println("First error:"); - errors.getFirst().printStackTrace(); - return 1; - } - - if (outputPropertiesPath != null) { - var properties = new Properties(); - properties.put("assets_root", assetRoot.toAbsolutePath().toString()); - properties.put("asset_index", assetIndexReference.id()); - try (var out = new BufferedOutputStream(Files.newOutputStream(Paths.get(outputPropertiesPath)))) { - properties.store(out, null); - } - } - - return 0; - } - - private static String getObjectPath(AssetObject object) { - var objectHash = object.hash(); - return objectHash.substring(0, 2) + "/" + objectHash; - } - - private class AssetDownloadSpec implements DownloadSpec { - private final AssetObject object; - - public AssetDownloadSpec(AssetObject object) { - this.object = object; - } - - @Override - public URI uri() { - return URI.create(assetRepository.toString() + getObjectPath(object)); - } - - @Override - public int size() { - return object.size(); - } - - @Override - public @Nullable String checksum() { - return object.hash(); - } - - @Override - public @Nullable String checksumAlgorithm() { - return "SHA1"; - } - } } diff --git a/src/main/java/net/neoforged/neoform/runtime/cli/LockManager.java b/src/main/java/net/neoforged/neoform/runtime/cli/LockManager.java index c0b5b2a..dff355d 100644 --- a/src/main/java/net/neoforged/neoform/runtime/cli/LockManager.java +++ b/src/main/java/net/neoforged/neoform/runtime/cli/LockManager.java @@ -12,7 +12,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.concurrent.locks.Lock; public class LockManager implements AutoCloseable { private static final Logger LOG = Logger.create(); diff --git a/src/main/java/net/neoforged/neoform/runtime/cli/Main.java b/src/main/java/net/neoforged/neoform/runtime/cli/Main.java index a4aa168..d9806d9 100644 --- a/src/main/java/net/neoforged/neoform/runtime/cli/Main.java +++ b/src/main/java/net/neoforged/neoform/runtime/cli/Main.java @@ -26,6 +26,10 @@ public class Main { @Option(names = "--home-dir", scope = ScopeType.INHERIT, description = "Where NFRT should store caches.") Path homeDir = getDefaultHomeDir(); + @Option(names = "--assets-dir", scope = ScopeType.INHERIT, description = "Where NFRT should store Minecraft client assets. Defaults to a subdirectory of the home-dir.") + @Nullable + Path assetsDir; + @Option(names = "--work-dir", scope = ScopeType.INHERIT, description = "Where temporary working directories are stored. Defaults to the subfolder 'work' in the NFRT home dir.") @Nullable Path workDir; @@ -130,12 +134,12 @@ public List getEffectiveRepositories() { } public CacheManager createCacheManager() throws IOException { - var cacheManager = new CacheManager(homeDir, getWorkDir()); + var cacheManager = new CacheManager(homeDir, assetsDir, getWorkDir()); cacheManager.setVerbose(verbose); return cacheManager; } - public LauncherInstallations createLauncherInstallations() { + public LauncherInstallations createLauncherInstallations() throws IOException { var installations = new LauncherInstallations(launcherDirs); installations.setVerbose(verbose); return installations; diff --git a/src/main/java/net/neoforged/neoform/runtime/downloads/AssetDownloadResult.java b/src/main/java/net/neoforged/neoform/runtime/downloads/AssetDownloadResult.java new file mode 100644 index 0000000..b5e288f --- /dev/null +++ b/src/main/java/net/neoforged/neoform/runtime/downloads/AssetDownloadResult.java @@ -0,0 +1,40 @@ +package net.neoforged.neoform.runtime.downloads; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; + +public record AssetDownloadResult(Path assetRoot, String assetIndexId) { + + public void writeAsProperties(Path destination) throws IOException { + if (destination.getParent() != null) { + Files.createDirectories(destination.getParent()); + } + + var properties = new Properties(); + properties.put("assets_root", assetRoot.toAbsolutePath().toString()); + properties.put("asset_index", assetIndexId); + try (var out = new BufferedOutputStream(Files.newOutputStream(destination))) { + properties.store(out, null); + } + } + + public void writeAsJson(Path destination) throws IOException { + if (destination.getParent() != null) { + Files.createDirectories(destination.getParent()); + } + + var jsonObject = new JsonObject(); + jsonObject.addProperty("assets", assetRoot.toAbsolutePath().toString()); + jsonObject.addProperty("asset_index", assetIndexId); + var jsonString = new Gson().toJson(jsonObject); + Files.writeString(destination, jsonString, StandardCharsets.UTF_8); + } + +} diff --git a/src/main/java/net/neoforged/neoform/runtime/downloads/AssetDownloader.java b/src/main/java/net/neoforged/neoform/runtime/downloads/AssetDownloader.java new file mode 100644 index 0000000..92b50bb --- /dev/null +++ b/src/main/java/net/neoforged/neoform/runtime/downloads/AssetDownloader.java @@ -0,0 +1,174 @@ +package net.neoforged.neoform.runtime.downloads; + +import net.neoforged.neoform.runtime.artifacts.ArtifactManager; +import net.neoforged.neoform.runtime.cache.CacheManager; +import net.neoforged.neoform.runtime.cache.LauncherInstallations; +import net.neoforged.neoform.runtime.manifests.AssetIndex; +import net.neoforged.neoform.runtime.manifests.AssetIndexReference; +import net.neoforged.neoform.runtime.manifests.AssetObject; +import net.neoforged.neoform.runtime.manifests.MinecraftVersionManifest; +import net.neoforged.neoform.runtime.utils.Logger; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HexFormat; + +/** + * Downloads the client-side assets necessary to run Minecraft. + * Since Minecraft versions reuse various assets, Mojang has organized the assets into a sort of repository, + * where an asset index maps a relative path to an asset unique identified by its content hash. The same + * asset can be stored only once on disk and reused many times across Minecraft versions. + * The assets stored this way are called "objects", while the JSON files describing the mapping of + * paths to objects are called "asset index". + *

+ * On disk, an asset root is a directory that contains a subfolder containing asset index files ("indexes"), + * and a subfolder containing the actual objects ("objects"). + *

+ * The objects subfolder is further subdivided into 256 subfolders, each representing the first two characters + * of a file content hash. Each of these subfolders will contain the actual objects whose hash starts with the + * same characters as the folder name. + * Example: {@code objects/af/af96f55a90eaf11b327f1b5f8834a051027dc506}, which is one of the Minecraft icon files. + */ +public class AssetDownloader { + private static final Logger LOG = Logger.create(); + + private static final String INDEX_FOLDER = "indexes"; + + private static final String OBJECT_FOLDER = "objects"; + + private final DownloadManager downloadManager; + private final ArtifactManager artifactManager; + private final LauncherInstallations launcherInstallations; + private final CacheManager cacheManager; + + public AssetDownloader(DownloadManager downloadManager, + ArtifactManager artifactManager, + LauncherInstallations launcherInstallations, + CacheManager cacheManager) { + this.downloadManager = downloadManager; + this.artifactManager = artifactManager; + this.launcherInstallations = launcherInstallations; + this.cacheManager = cacheManager; + } + + public AssetDownloadResult downloadAssets(String minecraftVersion, + URI assetRepository, + boolean useLauncherAssetRoot, + boolean copyLauncherAssets, + int concurrentDownloads) throws IOException, DownloadsFailedException { + + var versionManifest = MinecraftVersionManifest.from(artifactManager.getVersionManifest(minecraftVersion).path()); + var assetIndexReference = versionManifest.assetIndex(); + LOG.println("Downloading asset index " + assetIndexReference.id()); + + var assetRoot = selectAssetRoot(useLauncherAssetRoot, assetIndexReference); + prepareAssetRoot(assetRoot); + + var assetIndex = acquireAssetIndex(assetRoot, assetIndexReference); + + var objectsFolder = assetRoot.resolve(OBJECT_FOLDER); + var objectsToDownload = assetIndex.objects().values().stream() + .distinct() // The same object can be referenced multiple times + .filter(obj -> { + var f = objectsFolder.resolve(obj.getRelativePath()).toFile(); + return f.length() != obj.size() || obj.size() == 0 && !f.exists(); + }) + .toList(); + + try (var downloader = new ParallelDownloader(downloadManager, concurrentDownloads, objectsFolder, objectsToDownload.size())) { + if (copyLauncherAssets) { + var objectDirectories = launcherInstallations.getAssetRoots() + .stream() + .map(d -> d.resolve("objects")) + .toList(); + + downloader.setLocalSources(objectDirectories); + } + + for (var object : objectsToDownload) { + var spec = new AssetDownloadSpec(assetRepository, object); + var objectHash = object.hash(); + String objectPath = objectHash.substring(0, 2) + "/" + objectHash; + downloader.submitDownload(spec, objectPath); + } + } + + return new AssetDownloadResult(assetRoot, assetIndexReference.id()); + } + + private AssetIndex acquireAssetIndex(Path assetRoot, AssetIndexReference assetIndexReference) throws IOException { + var indexFolder = assetRoot.resolve(INDEX_FOLDER); + var assetIndexPath = indexFolder.resolve(assetIndexReference.id() + ".json"); + downloadManager.download(assetIndexReference, assetIndexPath); + return AssetIndex.from(assetIndexPath); + } + + private Path selectAssetRoot(boolean useLauncherAssetRoot, AssetIndexReference assetIndexReference) { + Path assetRoot = null; + if (useLauncherAssetRoot) { + // We already may have an asset root with specifically the index we're looking for, + // and it might not be the launcher directory with otherwise the most indices + assetRoot = launcherInstallations.getAssetDirectoryForIndex(assetIndexReference.id()); + if (assetRoot == null) { + var assetRoots = launcherInstallations.getAssetRoots(); + if (!assetRoots.isEmpty()) { + assetRoot = assetRoots.getFirst(); + } + } + } + if (assetRoot == null) { + assetRoot = cacheManager.getAssetsDir(); + } + + LOG.println("Using Minecraft asset root: " + assetRoot); + return assetRoot; + } + + private static void prepareAssetRoot(Path assetRoot) throws IOException { + + var indexFolder = assetRoot.resolve(INDEX_FOLDER); + Files.createDirectories(indexFolder); + var objectsFolder = assetRoot.resolve(OBJECT_FOLDER); + Files.createDirectories(objectsFolder); + + // Pre-create all folders + for (var i = 0; i < 256; i++) { + var objectSubFolder = objectsFolder.resolve(HexFormat.of().toHexDigits(i, 2)); + Files.createDirectories(objectSubFolder); + } + + } + + private static class AssetDownloadSpec implements DownloadSpec { + private final URI assetsBaseUrl; + private final AssetObject object; + + public AssetDownloadSpec(URI assetsBaseUrl, AssetObject object) { + this.assetsBaseUrl = assetsBaseUrl; + this.object = object; + } + + @Override + public URI uri() { + return URI.create(assetsBaseUrl.toString() + object.getRelativePath()); + } + + @Override + public int size() { + return object.size(); + } + + @Override + public @Nullable String checksum() { + return object.hash(); + } + + @Override + public @Nullable String checksumAlgorithm() { + return "SHA1"; + } + } +} diff --git a/src/main/java/net/neoforged/neoform/runtime/downloads/DownloadManager.java b/src/main/java/net/neoforged/neoform/runtime/downloads/DownloadManager.java index f743906..a790f51 100644 --- a/src/main/java/net/neoforged/neoform/runtime/downloads/DownloadManager.java +++ b/src/main/java/net/neoforged/neoform/runtime/downloads/DownloadManager.java @@ -32,17 +32,20 @@ public class DownloadManager implements AutoCloseable { @Override public void close() throws Exception { + httpClient.shutdownNow(); httpClient.close(); executor.shutdownNow(); - executor.awaitTermination(1, TimeUnit.MINUTES); + if (!executor.awaitTermination(1, TimeUnit.MINUTES)) { + LOG.println("Failed to wait for background downloads to finish."); + } } public void download(URI uri, Path finalLocation) throws IOException { download(new SimpleDownloadSpec(uri), finalLocation); } - public void download(DownloadSpec spec, Path finalLocation) throws IOException { - download(spec, finalLocation, false); + public boolean download(DownloadSpec spec, Path finalLocation) throws IOException { + return download(spec, finalLocation, false); } public boolean download(DownloadSpec spec, Path finalLocation, boolean silent) throws IOException { @@ -64,22 +67,10 @@ public boolean download(DownloadSpec spec, Path finalLocation, boolean silent) t var partialFile = finalLocation.resolveSibling(finalLocation.getFileName() + "." + Math.random() + ".dltmp"); Files.createDirectories(partialFile.getParent()); - if (url.getScheme().equals("file")) { - // File system download (e.g. from maven local) - var fileInRepo = Path.of(url); - if (!Files.exists(fileInRepo)) { - throw new FileNotFoundException("Could not find: " + url); - } - } - try { if (url.getScheme().equals("file")) { // File system download (e.g. from maven local) var fileInRepo = Path.of(url); - if (!Files.exists(fileInRepo)) { - throw new FileNotFoundException("Could not find: " + url); - } - Files.copy(fileInRepo, partialFile, StandardCopyOption.REPLACE_EXISTING); } else { var request = HttpRequest.newBuilder(url) @@ -107,7 +98,7 @@ public boolean download(DownloadSpec spec, Path finalLocation, boolean silent) t } else if (response.statusCode() == 404) { throw new FileNotFoundException(url.toString()); } else { - lastError = new IOException("Failed to download " + url + ": " + response.statusCode()); + lastError = new IOException("Failed to download " + url + ": HTTP Status Code " + response.statusCode()); if (canRetryStatusCode(response.statusCode())) { waitForRetry(response); continue; @@ -124,7 +115,7 @@ public boolean download(DownloadSpec spec, Path finalLocation, boolean silent) t if (spec.size() != -1) { var fileSize = Files.size(partialFile); if (fileSize != spec.size()) { - throw new IOException("Size of downloaded file has unexpected size. (actual: " + fileSize + ", expected: " + spec.size() + ")"); + throw new IOException("Downloaded file has unexpected size. (actual: " + fileSize + ", expected: " + spec.size() + ")"); } } diff --git a/src/main/java/net/neoforged/neoform/runtime/downloads/DownloadsFailedException.java b/src/main/java/net/neoforged/neoform/runtime/downloads/DownloadsFailedException.java new file mode 100644 index 0000000..826214c --- /dev/null +++ b/src/main/java/net/neoforged/neoform/runtime/downloads/DownloadsFailedException.java @@ -0,0 +1,15 @@ +package net.neoforged.neoform.runtime.downloads; + +import java.util.List; + +public class DownloadsFailedException extends Exception { + private final List errors; + + public DownloadsFailedException(List errors) { + this.errors = errors; + } + + public List getErrors() { + return errors; + } +} diff --git a/src/main/java/net/neoforged/neoform/runtime/downloads/ParallelDownloader.java b/src/main/java/net/neoforged/neoform/runtime/downloads/ParallelDownloader.java new file mode 100644 index 0000000..14b72b5 --- /dev/null +++ b/src/main/java/net/neoforged/neoform/runtime/downloads/ParallelDownloader.java @@ -0,0 +1,149 @@ +package net.neoforged.neoform.runtime.downloads; + +import net.neoforged.neoform.runtime.utils.FileUtil; +import net.neoforged.neoform.runtime.utils.Logger; +import net.neoforged.neoform.runtime.utils.StringUtil; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * This class is capable of download a large number of files concurrently, while observing a maximum + * concurrent download limit. + */ +public class ParallelDownloader implements AutoCloseable { + private static final Logger LOG = Logger.create(); + private static final ThreadFactory DOWNLOAD_THREAD_FACTORY = Thread.ofVirtual().name("parallel-download", 1).factory(); + + private final DownloadManager downloadManager; + private final Semaphore semaphore; + @Nullable + private final ExecutorService executor; + private final AtomicInteger downloadsDone = new AtomicInteger(); + private final AtomicInteger copiesDone = new AtomicInteger(); + private final AtomicLong bytesDownloaded = new AtomicLong(); + private final AtomicLong bytesCopied = new AtomicLong(); + private final List errors = new ArrayList<>(); + private final Path destination; + private final int estimatedTotal; + private volatile List localSources = List.of(); + + public ParallelDownloader(DownloadManager downloadManager, int concurrentDownloads, Path destination, int estimatedTotal) { + this.downloadManager = downloadManager; + this.destination = destination; + this.estimatedTotal = estimatedTotal; + if (concurrentDownloads < 1) { + throw new IllegalStateException("Cannot set concurrent downloads to less than 1: " + concurrentDownloads); + } else if (concurrentDownloads == 1) { + executor = null; + semaphore = null; + } else { + executor = Executors.newThreadPerTaskExecutor(DOWNLOAD_THREAD_FACTORY); + semaphore = new Semaphore(concurrentDownloads); + } + } + + public void setLocalSources(List localSources) { + // Remove the destination if it's present in the local sources + if (localSources.contains(destination)) { + localSources = new ArrayList<>(localSources); + localSources.remove(destination); + } + this.localSources = localSources; + } + + public void submitDownload(DownloadSpec spec, String relativeDestination) throws DownloadsFailedException { + if (executor != null && semaphore != null) { + executor.execute(() -> { + boolean hasAcquired = false; + try { + semaphore.acquire(); + hasAcquired = true; + download(spec, relativeDestination); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + synchronized (errors) { + errors.add(e); + } + } finally { + if (hasAcquired) { + semaphore.release(); + } + } + }); + } else { + // Synchronously download if concurrentDownloads == 1 + try { + download(spec, relativeDestination); + } catch (IOException e) { + throw new DownloadsFailedException(List.of(e)); + } + } + } + + private void download(DownloadSpec spec, String relativePath) throws IOException { + var objectDestination = destination.resolve(relativePath); + + try { + // Check if the object may exist already + for (var localSource : localSources) { + var existingFile = localSource.resolve(relativePath); + if (Files.isRegularFile(existingFile) && Files.size(existingFile) == spec.size()) { + FileUtil.safeCopy(existingFile, objectDestination); + bytesCopied.addAndGet(Files.size(objectDestination)); + copiesDone.incrementAndGet(); + return; + } + } + + if (downloadManager.download(spec, objectDestination, true)) { + bytesDownloaded.addAndGet(spec.size()); + } + } finally { + var finished = downloadsDone.incrementAndGet(); + if (finished % 100 == 0) { + LOG.println(finished + "/" + estimatedTotal + " downloads"); + } + } + } + + @Override + public void close() throws DownloadsFailedException { + // Wait for the executor to finish + if (executor != null) { + executor.shutdown(); + try { + while (!executor.awaitTermination(1, TimeUnit.MINUTES)) { + Thread.yield(); + } + } catch (InterruptedException e) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + throw new DownloadsFailedException(List.of(e)); + } + } + + if (downloadsDone.get() > 0) { + LOG.println("Downloaded " + downloadsDone.get() + " files with a total size of " + StringUtil.formatBytes(bytesDownloaded.get())); + } + if (copiesDone.get() > 0) { + LOG.println("Copied " + copiesDone.get() + " files with a total size of " + StringUtil.formatBytes(bytesCopied.get())); + } + + if (!errors.isEmpty()) { + throw new DownloadsFailedException(errors); + } + } +} diff --git a/src/main/java/net/neoforged/neoform/runtime/engine/NeoFormInterpolator.java b/src/main/java/net/neoforged/neoform/runtime/engine/NeoFormInterpolator.java index 20fcf1d..54c5c49 100644 --- a/src/main/java/net/neoforged/neoform/runtime/engine/NeoFormInterpolator.java +++ b/src/main/java/net/neoforged/neoform/runtime/engine/NeoFormInterpolator.java @@ -1,11 +1,7 @@ package net.neoforged.neoform.runtime.engine; -import net.neoforged.neoform.runtime.config.neoform.NeoFormDistConfig; -import net.neoforged.neoform.runtime.config.neoform.NeoFormStep; - import java.util.Set; import java.util.regex.Pattern; -import java.util.zip.ZipFile; /** * Interpolates tokens of the form {@code {token}} found in the argument lists of NeoForm functions. diff --git a/src/main/java/net/neoforged/neoform/runtime/manifests/AssetObject.java b/src/main/java/net/neoforged/neoform/runtime/manifests/AssetObject.java index b74268d..d9c8ac9 100644 --- a/src/main/java/net/neoforged/neoform/runtime/manifests/AssetObject.java +++ b/src/main/java/net/neoforged/neoform/runtime/manifests/AssetObject.java @@ -1,4 +1,7 @@ package net.neoforged.neoform.runtime.manifests; public record AssetObject(String hash, int size) { + public String getRelativePath() { + return hash.substring(0, 2) + "/" + hash; + } } diff --git a/src/main/java/net/neoforged/neoform/runtime/utils/FileUtil.java b/src/main/java/net/neoforged/neoform/runtime/utils/FileUtil.java index aa294e6..8a1b83a 100644 --- a/src/main/java/net/neoforged/neoform/runtime/utils/FileUtil.java +++ b/src/main/java/net/neoforged/neoform/runtime/utils/FileUtil.java @@ -90,11 +90,21 @@ private static void printLockingInfo(AccessDeniedException ex) { * @param destination The destination file * @throws IOException If an I/O error occurs */ - private static void atomicMoveIfPossible(final Path source, final Path destination) throws IOException { + private static void atomicMoveIfPossible(Path source, Path destination) throws IOException { try { Files.move(source, destination, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } catch (final AtomicMoveNotSupportedException ex) { Files.move(source, destination, StandardCopyOption.REPLACE_EXISTING); } } + + /** + * Copies the given source to a tmp-file in the destination folder and then performs an atomic move. + */ + public static void safeCopy(Path source, Path destination) throws IOException { + var suffix = ProcessHandle.current().pid() + "." + Thread.currentThread().threadId() + ".tmp"; + var tempDestination = destination.resolveSibling(destination.getFileName().toString() + suffix); + Files.copy(source, tempDestination, StandardCopyOption.REPLACE_EXISTING); + atomicMove(tempDestination, destination); + } } diff --git a/src/main/java/net/neoforged/neoform/runtime/utils/HashingUtil.java b/src/main/java/net/neoforged/neoform/runtime/utils/HashingUtil.java index 13227d6..89fce38 100644 --- a/src/main/java/net/neoforged/neoform/runtime/utils/HashingUtil.java +++ b/src/main/java/net/neoforged/neoform/runtime/utils/HashingUtil.java @@ -15,7 +15,15 @@ public final class HashingUtil { private HashingUtil() { } + public static String sha1(Path path) throws IOException { + return hashFile(path, "SHA-1"); + } + public static String sha1(String value) { + return sha1(value.getBytes(StandardCharsets.UTF_8)); + } + + public static String sha1(byte[] value) { MessageDigest digest; try { digest = MessageDigest.getInstance("SHA1"); @@ -23,7 +31,7 @@ public static String sha1(String value) { throw new RuntimeException(e); } - digest.update(value.getBytes(StandardCharsets.UTF_8)); + digest.update(value); return HexFormat.of().formatHex(digest.digest()); } diff --git a/src/test/java/net/neoforged/neoform/runtime/downloads/AssetDownloadResultTest.java b/src/test/java/net/neoforged/neoform/runtime/downloads/AssetDownloadResultTest.java new file mode 100644 index 0000000..8a9cb30 --- /dev/null +++ b/src/test/java/net/neoforged/neoform/runtime/downloads/AssetDownloadResultTest.java @@ -0,0 +1,59 @@ +package net.neoforged.neoform.runtime.downloads; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.Properties; + +import static org.assertj.core.api.Assertions.assertThat; + +class AssetDownloadResultTest { + + @TempDir + Path tempDir; + + AssetDownloadResult result; + + @BeforeEach + void setUp() { + result = new AssetDownloadResult(tempDir.resolve("äöäpäö bnlanil"), "123"); + } + + @Test + void testWriteAsProperties() throws Exception { + var tempFile = tempDir.resolve("asset.properties"); + result.writeAsProperties(tempFile); + + var p = new Properties(); + try (var in = Files.newInputStream(tempFile)) { + p.load(in); + } + assertThat(p).containsOnly( + Map.entry("asset_index", "123"), + Map.entry("assets_root", result.assetRoot().toString()) + ); + } + + @Test + void testWriteAsJson() throws Exception { + var tempFile = tempDir.resolve("asset.json"); + result.writeAsJson(tempFile); + + JsonObject o; + try (var in = Files.newBufferedReader(tempFile, StandardCharsets.UTF_8)) { + o = new Gson().fromJson(in, JsonObject.class); + } + + JsonObject expected = new JsonObject(); + expected.addProperty("asset_index", "123"); + expected.addProperty("assets", result.assetRoot().toString()); + assertThat(o).isEqualTo(expected); + } +} \ No newline at end of file diff --git a/src/test/java/net/neoforged/neoform/runtime/downloads/AssetDownloaderTest.java b/src/test/java/net/neoforged/neoform/runtime/downloads/AssetDownloaderTest.java new file mode 100644 index 0000000..d2f8e26 --- /dev/null +++ b/src/test/java/net/neoforged/neoform/runtime/downloads/AssetDownloaderTest.java @@ -0,0 +1,379 @@ +package net.neoforged.neoform.runtime.downloads; + +import com.google.gson.Gson; +import net.neoforged.neoform.runtime.artifacts.Artifact; +import net.neoforged.neoform.runtime.artifacts.ArtifactManager; +import net.neoforged.neoform.runtime.cache.CacheManager; +import net.neoforged.neoform.runtime.cache.LauncherInstallations; +import net.neoforged.neoform.runtime.manifests.AssetIndex; +import net.neoforged.neoform.runtime.manifests.AssetIndexReference; +import net.neoforged.neoform.runtime.manifests.AssetObject; +import net.neoforged.neoform.runtime.manifests.MinecraftVersionManifest; +import net.neoforged.neoform.runtime.utils.HashingUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; + +@MockitoSettings(strictness = Strictness.LENIENT) +class AssetDownloaderTest { + // A fake MC version we generate a version manifest for, which points to our mocked asset index + private static final String MC_VERSION = "1.2.3"; + // The asset index id we use for the generated MC version manifest + private static final String ASSET_INDEX_ID = "1234"; + private static final String BASE_URI = "http://assets.fake/assets/"; + + @TempDir + Path tempDir; + @TempDir + Path nfrtAssetsDir; + @Mock + ArtifactManager artifactManager; + @Mock + DownloadManager downloadManager; + @Mock + CacheManager cacheManager; + @Mock + LauncherInstallations launcherInstallations; + + @InjectMocks + AssetDownloader downloader; + + private int assetCounter; + + // Relative URI -> Content + Map downloadableContent = new HashMap<>(); + + // Actual downloads + List downloadedRelativePaths = new ArrayList<>(); + private Path versionManifestPath; + + @BeforeEach + void setUp() throws IOException { + nfrtAssetsDir = tempDir.resolve("assets"); + when(cacheManager.getAssetsDir()).thenReturn(nfrtAssetsDir); + + versionManifestPath = tempDir.resolve("minecraft_version.json"); + + // Mock the download service + doAnswer(invocation -> { + var spec = invocation.getArgument(0, DownloadSpec.class); + assertThat(spec.uri().toString()).startsWith(BASE_URI); + var relative = spec.uri().toString().substring(BASE_URI.length()); + var content = downloadableContent.get(relative); + if (content == null) { + throw new RuntimeException("Unknown URI: " + relative); + } + var destination = invocation.getArgument(1, Path.class); + Files.write(destination, content); + downloadedRelativePaths.add(relative); + return null; + }).when(downloadManager).download(any(DownloadSpec.class), any(Path.class)); + doAnswer(invocation -> { + var spec = invocation.getArgument(0, DownloadSpec.class); + assertThat(spec.uri().toString()).startsWith(BASE_URI); + var relative = spec.uri().toString().substring(BASE_URI.length()); + var content = downloadableContent.get(relative); + if (content == null) { + throw new RuntimeException("Unknown URI: " + relative); + } + var destination = invocation.getArgument(1, Path.class); + Files.write(destination, content); + downloadedRelativePaths.add(relative); + return false; + }).when(downloadManager).download(any(DownloadSpec.class), any(Path.class), anyBoolean()); + + when(artifactManager.getVersionManifest(MC_VERSION)) + .thenAnswer(invoc -> Artifact.ofPath(versionManifestPath)); + } + + @Test + void testDownloadWithoutLauncherInstallation() throws Exception { + setAssetIndex(new AssetIndex(Map.of())); + downloader.downloadAssets( + MC_VERSION, + URI.create(BASE_URI), + false, + false, + 2 + ); + } + + @Test + void inAbsenceOfLaunchersAssetDirPointsToNfrtAssetDir() throws Exception { + setAssetIndex(new AssetIndex(Map.of())); + var result = downloader.downloadAssets( + MC_VERSION, + URI.create(BASE_URI), + false, + false, + 2 + ); + assertEquals(nfrtAssetsDir, result.assetRoot()); + } + + @Test + void resultAssetIndexExists() throws Exception { + setAssetIndex(new AssetIndex(Map.of())); + var result = downloader.downloadAssets( + MC_VERSION, + URI.create(BASE_URI), + false, + false, + 2 + ); + assertThat(result.assetRoot().resolve("indexes/" + result.assetIndexId() + ".json")).isRegularFile(); + } + + @Test + void testDownloadOfMultipleAssets() throws Exception { + var assetPaths = List.of( + "rootasset", + "rootasset.ext", + "first_level/asset", + "first_level/sub_level/asset", + "direct/third/level/asset" + ); + var assetIndex = generateAssetIndex(assetPaths); + var result = downloader.downloadAssets( + MC_VERSION, + URI.create(BASE_URI), + false, + false, + 2 + ); + + for (var asset : assetIndex.objects().values()) { + var targetPath = result.assetRoot().resolve("objects").resolve(asset.getRelativePath()); + assertThat(targetPath) + .isRegularFile() + .hasDigest("SHA-1", asset.hash()); + } + } + + @Test + void testDownloadFailuresAreCollectedAndThrownAtEndWhenDownloadingConcurrently() throws Exception { + generateAssetIndex(List.of( + "asset1", + "asset2" + )); + doThrow(new RuntimeException("exc1"), new RuntimeException("exc2")).when(downloadManager) + .download(any(), any(), anyBoolean()); + + var e = assertThrows(DownloadsFailedException.class, () -> downloader.downloadAssets( + MC_VERSION, + URI.create(BASE_URI), + false, + false, + 2 + )); + assertThat(e.getErrors()).hasSize(2); + } + + @Test + void testCorruptRemoteFileIsValidated() throws Exception { + generateAssetIndex(List.of( + "asset1", + "asset2" + )); + doThrow(new RuntimeException("exc1"), new RuntimeException("exc2")).when(downloadManager) + .download(any(), any(), anyBoolean()); + + var e = assertThrows(DownloadsFailedException.class, () -> downloader.downloadAssets( + MC_VERSION, + URI.create(BASE_URI), + false, + false, + 2 + )); + assertThat(e.getErrors()).hasSize(2); + } + + @Nested + class ReuseOfAssetsFromLaunchers { + private AssetIndex assetIndex; + private Path fakeLauncher1; + private Path fakeLauncher2; + private AssetObject asset1; + private AssetObject asset2; + + @BeforeEach + void setUp() throws IOException { + var assetPaths = List.of("a/b/asset", "b/c/asset"); + assetIndex = generateAssetIndex(assetPaths); + asset1 = assetIndex.objects().get("a/b/asset"); + asset2 = assetIndex.objects().get("b/c/asset"); + + fakeLauncher1 = tempDir.resolve("fakeLauncher1"); + fakeLauncher2 = tempDir.resolve("fakeLauncher2"); + when(launcherInstallations.getAssetRoots()).thenReturn(List.of(fakeLauncher1, fakeLauncher2)); + + // The first asset is present in launcher1, but is corrupted + writeFile(fakeLauncher1.resolve("objects/" + asset1.getRelativePath()), "THIS IS NOT THE RIGHT CONTENT".getBytes()); + // The first asset is also present in launcher2, but with the right content + writeFile(fakeLauncher2.resolve("objects/" + asset1.getRelativePath()), downloadableContent.get(asset1.getRelativePath())); + // asset2 is present in neither launcher + } + + @Test + void testCopyingFromLauncherIsDisabled() throws Exception { + var result = runDownloadAndValidateResult(false, false); + + // The returned root should be the NFRT root + assertEquals(nfrtAssetsDir, result.assetRoot()); + + // It should have downloaded both assets via the download manager + assertThat(downloadedRelativePaths).containsExactlyInAnyOrder( + "asset_index.json", asset1.getRelativePath(), asset2.getRelativePath() + ); + } + + @Test + void testCopyingAsset1FromLauncherDir2() throws Exception { + var result = runDownloadAndValidateResult(false, true); + + // The returned root should be the NFRT root + assertEquals(nfrtAssetsDir, result.assetRoot()); + + // It should have downloaded both assets via the download manager + assertThat(downloadedRelativePaths).containsExactlyInAnyOrder( + "asset_index.json", asset2.getRelativePath() + ); + } + + @Test + void testReusingFirstAvailableLauncherAssetRoot() throws Exception { + var result = runDownloadAndValidateResult(true, false); + + assertEquals(fakeLauncher1, result.assetRoot()); + + // It should have downloaded both assets via the download manager + // because while asset 1 is present in launcher1 asset folder, it has an unexpected size + assertThat(downloadedRelativePaths).containsExactlyInAnyOrder( + "asset_index.json", + asset1.getRelativePath(), + asset2.getRelativePath() + ); + } + + @Test + void testReusingLauncherAssetRootThatAlreadyHasTheAssetIndex() throws Exception { + when(launcherInstallations.getAssetDirectoryForIndex(ASSET_INDEX_ID)).thenReturn(fakeLauncher2); + + var result = runDownloadAndValidateResult(true, false); + + assertEquals(fakeLauncher2, result.assetRoot()); + + // In the asset root of launcher 2, asset 1 exists and is already valid + assertThat(downloadedRelativePaths).containsExactlyInAnyOrder( + "asset_index.json", + asset2.getRelativePath() + ); + } + + private AssetDownloadResult runDownloadAndValidateResult(boolean useLauncherRoot, boolean copyFromLauncher) throws Exception { + var result = downloader.downloadAssets( + MC_VERSION, + URI.create(BASE_URI), + useLauncherRoot, + copyFromLauncher, + 1 + ); + validateAssetDownloadResult(result, assetIndex); + return result; + } + } + + private AssetIndex generateAssetIndex(List assetPaths) throws IOException { + var objects = new HashMap(); + for (String assetPath : assetPaths) { + objects.put(assetPath, generateAsset()); + } + var assetIndex = new AssetIndex(objects); + setAssetIndex(assetIndex); + return assetIndex; + } + + private void setAssetIndex(AssetIndex assetIndex) throws IOException { + var json = new Gson().toJson(assetIndex); + downloadableContent.put("asset_index.json", json.getBytes(StandardCharsets.UTF_8)); + + var assetIndexRef = new AssetIndexReference( + ASSET_INDEX_ID, + HashingUtil.sha1(json), + json.length(), + 0, + URI.create(BASE_URI + "asset_index.json") + ); + var manifest = new MinecraftVersionManifest( + MC_VERSION, + Map.of(), + List.of(), + assetIndexRef, + "", + null, + null, + null + ); + Files.writeString(versionManifestPath, new Gson().toJson(manifest)); + } + + private AssetObject generateAsset() { + var assetId = ++assetCounter; + var r = new Random(123); + for (int i = 0; i < assetId; i++) { + r.nextLong(); + } + var assetR = new Random(r.nextLong()); + var size = assetR.nextInt(65535); + var content = new byte[size]; + r.nextBytes(content); + + var sha1 = HashingUtil.sha1(content); + var assetObject = new AssetObject(sha1, content.length); + downloadableContent.put(assetObject.getRelativePath(), content); + return assetObject; + } + + private static void validateAssetDownloadResult(AssetDownloadResult result, AssetIndex expectedIndex) { + // Just validates that both assets were downloaded correctly + for (var entry : expectedIndex.objects().entrySet()) { + var asset = entry.getValue(); + var targetPath = result.assetRoot().resolve("objects").resolve(asset.getRelativePath()); + assertThat(targetPath) + .describedAs("Original path: %s", entry.getKey()) + .isRegularFile() + .hasDigest("SHA-1", asset.hash()); + } + } + + private static void writeFile(Path p, byte[] content) throws IOException { + Files.createDirectories(p.getParent()); + Files.write(p, content); + } + +} diff --git a/src/test/java/net/neoforged/neoform/runtime/downloads/DownloadManagerTest.java b/src/test/java/net/neoforged/neoform/runtime/downloads/DownloadManagerTest.java new file mode 100644 index 0000000..c86dc73 --- /dev/null +++ b/src/test/java/net/neoforged/neoform/runtime/downloads/DownloadManagerTest.java @@ -0,0 +1,239 @@ +package net.neoforged.neoform.runtime.downloads; + +import com.sun.net.httpserver.Filter; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import com.sun.net.httpserver.SimpleFileServer; +import net.neoforged.neoform.runtime.utils.HashingUtil; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.Inet4Address; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DownloadManagerTest { + @TempDir + Path tempDir; + @TempDir + Path remoteWebRoot; + HttpServer server; + String baseUrl; + DownloadManager downloadManager = new DownloadManager(); + List requests = new ArrayList<>(); + List queuedErrors = new ArrayList<>(); + + @BeforeEach + void setUp() throws Exception { + InetSocketAddress addr = new InetSocketAddress(Inet4Address.getLocalHost(), 0); + var fileHandler = SimpleFileServer.createFileHandler(remoteWebRoot); + server = HttpServer.create(addr, 1, "/", fileHandler, new Filter() { + @Override + public void doFilter(HttpExchange exchange, Chain chain) throws IOException { + requests.add(exchange.getRequestURI().toString()); + if (!queuedErrors.isEmpty()) { + exchange.sendResponseHeaders(queuedErrors.removeFirst(), 0); + exchange.close(); + return; + } + chain.doFilter(exchange); + } + + @Override + public String description() { + return "logging filter"; + } + }); + + server.start(); + var address = server.getAddress(); + baseUrl = "http://" + address.getAddress().getHostAddress() + ":" + address.getPort(); + } + + @AfterEach + void tearDown() throws Exception { + server.stop(0); + downloadManager.close(); + } + + @Test + void testSimpleDownload() throws IOException { + var uri = URI.create(baseUrl + "/testpath.dat"); + Files.writeString(remoteWebRoot.resolve("testpath.dat"), "hello, world!"); + var destination = tempDir.resolve("test.dat"); + downloadManager.download(uri, destination); + assertThat(destination).hasContent("hello, world!"); + } + + /** + * If only a URI is provided, and no length or checksum, the downloader will always re-download. + */ + @Test + void testSimpleDownloadUnconditionallyOverwrites() throws Exception { + var uri = URI.create(baseUrl + "/testpath.dat"); + Files.writeString(remoteWebRoot.resolve("testpath.dat"), "hello, world!"); + var destination = tempDir.resolve("test.dat"); + downloadManager.download(uri, destination); + downloadManager.download(uri, destination); + assertThat(destination).hasContent("hello, world!"); + assertThat(requests).containsExactly("/testpath.dat", "/testpath.dat"); + } + + /** + * If SHA-1 checksum and length are provided, the downloader will first check if the + * file is already downloaded. + */ + @Test + void testValidExistingFilesAreNotRedownloaded() throws Exception { + var remoteFile = remoteWebRoot.resolve("testpath.dat"); + var destination = tempDir.resolve("test.dat"); + Files.writeString(remoteFile, "hello, world!"); + Files.copy(remoteFile, destination); + + assertFalse(downloadManager.download(downloadSpecFor(remoteFile), destination)); + assertThat(destination).hasContent("hello, world!"); + assertThat(requests).isEmpty(); + } + + /** + * If SHA-1 checksum and length are provided, and the local file is corrupted, + * it will be re-downloaded. + */ + @Test + void testCorruptedLocalFilesAreRedownloaded() throws Exception { + var remoteFile = remoteWebRoot.resolve("testpath.dat"); + var destination = tempDir.resolve("test.dat"); + Files.writeString(remoteFile, "hello, world!"); + var downloadSpec = downloadSpecFor(remoteFile); + + Files.writeString(destination, "CORRUPTED!"); + + assertTrue(downloadManager.download(downloadSpec, destination)); + + assertThat(destination).hasContent("hello, world!"); + assertThat(requests).containsExactly("/testpath.dat"); + } + + /** + * If length is provided, the file downloaded from the remote is validated. + */ + @Test + void testCorruptedRemoteFilesAreRejectedBasedOnSize() throws Exception { + var remoteFile = remoteWebRoot.resolve("testpath.dat"); + var destination = tempDir.resolve("test.dat"); + Files.writeString(remoteFile, "hello, world!"); + var downloadSpec = new FullDownloadSpec(URI.create(baseUrl + "/testpath.dat"), (int) Files.size(remoteFile), null); + Files.writeString(remoteFile, "and now it is corrupted because its size differs!!!"); + + var e = assertThrows(IOException.class, () -> downloadManager.download(downloadSpec, destination)); + assertThat(e).hasMessageContaining("Downloaded file has unexpected size. (actual: 51, expected: 13)"); + } + + /** + * If SHA-1 checksum is provided, the file downloaded from the remote is validated. + */ + @Test + void testCorruptedRemoteFilesAreRejectedBasedOnChecksum() throws Exception { + var remoteFile = remoteWebRoot.resolve("testpath.dat"); + var destination = tempDir.resolve("test.dat"); + Files.writeString(remoteFile, "hello, world!"); + var downloadSpec = new FullDownloadSpec(URI.create(baseUrl + "/testpath.dat"), -1, HashingUtil.sha1(remoteFile)); + Files.writeString(remoteFile, "hello, warld!"); + + var e = assertThrows(IOException.class, () -> downloadManager.download(downloadSpec, destination)); + assertThat(e).hasMessageContaining("Downloaded file has unexpected checksum. (actual: decdbe49afb7782c8a07b7750097e77ecf73f437, expected: 1f09d30c707d53f3d16c530dd73d70a6ce7596a9)"); + } + + @Test + void testStatusCodeIsRetried() throws Exception { + queuedErrors.add(429); + + var remoteFile = remoteWebRoot.resolve("testpath.dat"); + var destination = tempDir.resolve("test.dat"); + Files.writeString(remoteFile, "hello, world!"); + assertTrue(downloadManager.download(downloadSpecFor(remoteFile), destination)); + assertThat(requests).containsExactly("/testpath.dat", "/testpath.dat"); + } + + @Test + void testRetryOnStatusCodeStopsAfterFiveTries() throws Exception { + Collections.addAll(queuedErrors, 429, 429, 429, 429, 429, 429, 429, 429, 429, 429, 429, 429); + + var remoteFile = remoteWebRoot.resolve("testpath.dat"); + var destination = tempDir.resolve("test.dat"); + Files.writeString(remoteFile, "hello, world!"); + var e = assertThrows(IOException.class, () -> downloadManager.download(downloadSpecFor(remoteFile), destination)); + assertThat(e) + .hasMessageContaining("Failed to download") + .hasMessageContaining("HTTP Status Code 429"); + assertThat(requests).containsExactly( + "/testpath.dat", + "/testpath.dat", + "/testpath.dat", + "/testpath.dat", + "/testpath.dat" + ); + } + + @Test + void testFileNotFoundIsNotRetried() throws Exception { + Collections.addAll(queuedErrors, 404); + + var remoteFile = remoteWebRoot.resolve("testpath.dat"); + var destination = tempDir.resolve("test.dat"); + Files.writeString(remoteFile, "hello, world!"); + assertThrows(FileNotFoundException.class, () -> downloadManager.download(downloadSpecFor(remoteFile), destination)); + assertThat(requests).containsExactly("/testpath.dat"); + } + + @Test + void testServerErrorsAreNotRetried() throws Exception { + Collections.addAll(queuedErrors, 500, 500, 500); + + var remoteFile = remoteWebRoot.resolve("testpath.dat"); + var destination = tempDir.resolve("test.dat"); + Files.writeString(remoteFile, "hello, world!"); + var e = assertThrows(IOException.class, () -> downloadManager.download(downloadSpecFor(remoteFile), destination)); + assertThat(e) + .hasMessageContaining("Failed to download") + .hasMessageContaining("HTTP Status Code 500"); + assertThat(requests).containsExactly("/testpath.dat"); + } + + @Test + void testSupportsFileUrlDownloads() throws IOException { + var remoteFile = remoteWebRoot.resolve("testpath.dat"); + Files.writeString(remoteFile, "hello, world!"); + var destination = tempDir.resolve("test.dat"); + downloadManager.download(remoteFile.toUri(), destination); + } + + FullDownloadSpec downloadSpecFor(Path path) throws IOException { + return new FullDownloadSpec( + URI.create(baseUrl + "/" + remoteWebRoot.relativize(path).toString().replace('\\', '/')), + (int) Files.size(path), + HashingUtil.sha1(path) + ); + } + + record FullDownloadSpec(URI uri, int size, String checksum) implements DownloadSpec { + @Override + public String checksumAlgorithm() { + return checksum != null ? "SHA-1" : null; + } + } +} \ No newline at end of file