Add support for local copy of dependency trees

This change is still in progress, currently by providing the argument
`--cache-dependencies-locally`, you can have grabbed dependencies stored
in a local `zig-deps` folder which theoretically should be viable for
checking in to your VCS.

However - there are a few unsolved problems:
  - dependenices which are no longer depended on should probably be
    removed.
  - dependencies being stored flatly in the zig-deps folder by hash
    makes it non-obvious which dependency is which (from a user POV).
    Additionally, if you updated dependencies the hash (and by extension
    folder name) will change - this then makes tracking logical changes
    to dependencies in your version control system trickier.

I'm not entirely sure how to best solve these problems - one could be to
have some kind of naming-resolution scheme, alongside a local lock file
mapping names to hashes? This could be combined with a build option to
force hashes to be recomputed to validate the contents of these folders
have not changed (but having it as an option as it would be slower).

Author: moosichu

lib/compiler/build_runner.zig

@@ -213,6 +213,8 @@ pub fn main() !void {
                 builder.verbose_link = true;
             } else if (mem.eql(u8, arg, "--verbose-air")) {
                 builder.verbose_air = true;
+            } else if (mem.eql(u8, arg, "--cache-dependencies-locally")) {
+                builder.cache_dependencies_locally = true;
             } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) {
                 builder.verbose_llvm_ir = "-";
             } else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) {
@@ -1184,6 +1186,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void {
         \\  --build-file [file]          Override path to build.zig
         \\  --cache-dir [path]           Override path to local Zig cache directory
         \\  --global-cache-dir [path]    Override path to global Zig cache directory
+        \\  --cache-dependencies-locally Cache dependencies locally in zig-deps
         \\  --zig-lib-dir [arg]          Override path to Zig lib directory
         \\  --build-runner [file]        Override path to build runner
         \\  --seed [integer]             For shuffling dependency traversal order (default: random)

lib/std/Build.zig

@@ -33,6 +33,7 @@ verbose: bool,
 verbose_link: bool,
 verbose_cc: bool,
 verbose_air: bool,
+cache_dependencies_locally: bool,
 verbose_llvm_ir: ?[]const u8,
 verbose_llvm_bc: ?[]const u8,
 verbose_cimport: bool,
@@ -254,6 +255,7 @@ pub fn create(
         .graph = graph,
         .build_root = build_root,
         .cache_root = cache_root,
+        .cache_dependencies_locally = false,
         .verbose = false,
         .verbose_link = false,
         .verbose_cc = false,
@@ -380,6 +382,7 @@ fn createChildOnly(
         .installed_files = ArrayList(InstalledFile).init(allocator),
         .build_root = build_root,
         .cache_root = parent.cache_root,
+        .cache_dependencies_locally = parent.cache_dependencies_locally,
         .zig_lib_dir = parent.zig_lib_dir,
         .debug_log_scopes = parent.debug_log_scopes,
         .debug_compile_errors = parent.debug_compile_errors,

lib/std/Build/Step/Compile.zig

@@ -1380,7 +1380,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
     if (b.debug_compile_errors) {
         try zig_args.append("--debug-compile-errors");
     }
-
+    if (b.cache_dependencies_locally) try zig_args.append("--cache-dependencies-locally");
     if (b.verbose_cimport) try zig_args.append("--verbose-cimport");
     if (b.verbose_air) try zig_args.append("--verbose-air");
     if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path}));

src/Package/Fetch.zig

@@ -1,9 +1,18 @@
 //! Represents one independent job whose responsibility is to:
 //!
-//! 1. Check the global zig package cache to see if the hash already exists.
-//!    If so, load, parse, and validate the build.zig.zon file therein, and
-//!    goto step 8. Likewise if the location is a relative path, treat this
-//!    the same as a cache hit. Otherwise, proceed.
+//! 1. if local zig packages is enabled:
+//!      a. Check the local zig package directory to see if the hash already exists.
+//!         If so, load, parse, and validate the build.zig.zon file therein, and
+//!         goto step 8. Likewise if the location is a relative path, treat this
+//!         the same as a cache hit. Otherwise, proceed.
+//!      b. Check the global zig package cache to see if the hash already exists.
+//!         If so, load, parse, and validate the build.zig.zon file therein, copy
+//!         it to the local zig package directory and goto step 8. Otherwise, proceed.
+//!    if local zig packages is disabled:
+//!      Check the global zig package cache to see if the hash already exists.
+//!      If so, load, parse, and validate the build.zig.zon file therein, and
+//!      goto step 8. Likewise if the location is a relative path, treat this
+//!      the same as a cache hit. Otherwise, proceed.
 //! 2. Fetch and unpack a URL into a temporary directory.
 //! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed
 //!    for the file to be missing, in which case this fetched package is considered
@@ -18,7 +27,8 @@
 //!    directory. If the hash already exists, delete the temporary directory and
 //!    leave the zig package cache directory untouched as it may be in use by the
 //!    system. This is done even if the hash is invalid, in case the package with
-//!    the different hash is used in the future.
+//!    the different hash is used in the future. If local packages is enabled, copy
+//!    the package locally.
 //! 7. Validate the computed hash against the expected hash. If invalid,
 //!    this job is done.
 //! 8. Spawn a new fetch job for each dependency in the manifest file. Use
@@ -97,6 +107,7 @@ pub const JobQueue = struct {
     thread_pool: *ThreadPool,
     wait_group: WaitGroup = .{},
     global_cache: Cache.Directory,
+    local_package_cache_root: ?Cache.Directory,
     /// If true then, no fetching occurs, and:
     /// * The `global_cache` directory is assumed to be the direct parent
     ///   directory of on-disk packages rather than having the "p/" directory
@@ -299,7 +310,7 @@ pub fn run(f: *Fetch) RunError!void {
     const arena = f.arena.allocator();
     const gpa = f.arena.child_allocator;
     const cache_root = f.job_queue.global_cache;
-
+    const local_package_cache = f.job_queue.local_package_cache_root;
     try eb.init(gpa);
 
     // Check the global zig package cache to see if the hash already exists. If
@@ -372,41 +383,102 @@ pub fn run(f: *Fetch) RunError!void {
         const prefixed_pkg_sub_path = "p" ++ s ++ expected_hash;
         const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
         const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
-        if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
-            assert(f.lazy_status != .unavailable);
-            f.package_root = .{
-                .root_dir = cache_root,
-                .sub_path = try arena.dupe(u8, pkg_sub_path),
-            };
-            try loadManifest(f, f.package_root);
-            try checkBuildFileExistence(f);
-            if (!f.job_queue.recursive) return;
-            return queueJobsForDeps(f);
-        } else |err| switch (err) {
-            error.FileNotFound => {
-                switch (f.lazy_status) {
-                    .eager => {},
-                    .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) {
-                        f.lazy_status = .unavailable;
-                        return;
-                    },
-                    .unavailable => unreachable,
+        var check_cache_root = local_package_cache == null;
+        if (local_package_cache) |lpc| {
+            if (lpc.handle.access(&expected_hash, .{})) {
+                f.package_root = .{
+                    .root_dir = lpc,
+                    .sub_path = try arena.dupe(u8, &expected_hash),
+                };
+                try loadManifest(f, f.package_root);
+                try checkBuildFileExistence(f);
+                if (!f.job_queue.recursive) return;
+                return queueJobsForDeps(f);
+            } else |err| switch (err) {
+                error.FileNotFound => {
+                    check_cache_root = true;
+                },
+                else => |e| {
+                    try eb.addRootErrorMessage(.{
+                        .msg = try eb.printString("unable to open local package cache directory '{}{s}': {s}", .{
+                            lpc, &expected_hash, @errorName(e),
+                        }),
+                    });
+                    return error.FetchFailed;
+                },
+            }
+        }
+        if (check_cache_root) {
+            if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
+                assert(f.lazy_status != .unavailable);
+                if (local_package_cache) |lpc| {
+                    var dst_handle = lpc.handle.makeOpenPath(&expected_hash, .{}) catch |err| {
+                        try eb.addRootErrorMessage(.{
+                            .msg = try eb.printString("unable to open local package cache directory '{}{s}': {s}", .{
+                                lpc, &expected_hash, @errorName(err),
+                            }),
+                        });
+                        return error.FetchFailed;
+                    };
+                    defer dst_handle.close();
+                    var src_handle = cache_root.handle.openDir(pkg_sub_path, .{}) catch |err| {
+                        try eb.addRootErrorMessage(.{
+                            .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{
+                                cache_root, pkg_sub_path, @errorName(err),
+                            }),
+                        });
+                        return error.FetchFailed;
+                    };
+                    defer src_handle.close();
+                    f.recursiveDirectoryCopy(src_handle, dst_handle) catch |err| {
+                        const src = try cache_root.join(arena, &.{pkg_sub_path});
+                        const dest = try lpc.join(arena, &.{&expected_hash});
+                        try eb.addRootErrorMessage(.{ .msg = try eb.printString(
+                            "sunable to copy into local package directory '{s}' from package cache directory '{s}': {s}",
+                            .{ dest, src, @errorName(err) },
+                        ) });
+                        return error.FetchFailed;
+                    };
+                    f.package_root = .{
+                        .root_dir = lpc,
+                        .sub_path = try arena.dupe(u8, &expected_hash),
+                    };
+                } else {
+                    f.package_root = .{
+                        .root_dir = cache_root,
+                        .sub_path = try arena.dupe(u8, pkg_sub_path),
+                    };
                 }
-                if (f.job_queue.read_only) return f.fail(
-                    f.name_tok,
-                    try eb.printString("package not found at '{}{s}'", .{
-                        cache_root, pkg_sub_path,
-                    }),
-                );
-            },
-            else => |e| {
-                try eb.addRootErrorMessage(.{
-                    .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{
-                        cache_root, pkg_sub_path, @errorName(e),
-                    }),
-                });
-                return error.FetchFailed;
-            },
+                try loadManifest(f, f.package_root);
+                try checkBuildFileExistence(f);
+                if (!f.job_queue.recursive) return;
+                return queueJobsForDeps(f);
+            } else |err| switch (err) {
+                error.FileNotFound => {
+                    switch (f.lazy_status) {
+                        .eager => {},
+                        .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) {
+                            f.lazy_status = .unavailable;
+                            return;
+                        },
+                        .unavailable => unreachable,
+                    }
+                    if (f.job_queue.read_only) return f.fail(
+                        f.name_tok,
+                        try eb.printString("package not found at '{}{s}'", .{
+                            cache_root, pkg_sub_path,
+                        }),
+                    );
+                },
+                else => |e| {
+                    try eb.addRootErrorMessage(.{
+                        .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{
+                            cache_root, pkg_sub_path, @errorName(e),
+                        }),
+                    });
+                    return error.FetchFailed;
+                },
+            }
         }
     } else if (f.job_queue.read_only) {
         try eb.addRootErrorMessage(.{
@@ -444,6 +516,7 @@ fn runResource(
     const eb = &f.error_bundle;
     const s = fs.path.sep_str;
     const cache_root = f.job_queue.global_cache;
+    const local_package_cache = f.job_queue.local_package_cache_root;
     const rand_int = std.crypto.random.int(u64);
     const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
 
@@ -526,6 +599,36 @@ fn runResource(
         ) });
         return error.FetchFailed;
     };
+    if (local_package_cache) |lpc| {
+        var dst_handle = lpc.handle.makeOpenPath(&Manifest.hexDigest(f.actual_hash), .{}) catch |err| {
+            try eb.addRootErrorMessage(.{
+                .msg = try eb.printString("unable to open local package cache directory '{}{s}': {s}", .{
+                    lpc, &Manifest.hexDigest(f.actual_hash), @errorName(err),
+                }),
+            });
+            return error.FetchFailed;
+        };
+        defer dst_handle.close();
+        var src_handle = cache_root.handle.openDir(f.package_root.sub_path, .{}) catch |err| {
+            try eb.addRootErrorMessage(.{
+                .msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{
+                    cache_root, f.package_root.sub_path, @errorName(err),
+                }),
+            });
+            return error.FetchFailed;
+        };
+        defer src_handle.close();
+        f.recursiveDirectoryCopy(src_handle, dst_handle) catch |err| {
+            const src = try cache_root.join(arena, &.{f.package_root.sub_path});
+            const dest = try lpc.join(arena, &.{&Manifest.hexDigest(f.actual_hash)});
+            try eb.addRootErrorMessage(.{ .msg = try eb.printString(
+                "unable to copy into local package directory '{s}' from package cache directory '{s}': {s}",
+                .{ dest, src, @errorName(err) },
+            ) });
+            return error.FetchFailed;
+        };
+    }
+
     // Remove temporary directory root if not already renamed to global cache.
     if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) {
         cache_root.handle.deleteDir(tmp_dir_sub_path) catch {};

src/main.zig

@@ -4632,6 +4632,7 @@ const usage_build =
     \\  --build-file [file]           Override path to build.zig
     \\  --cache-dir [path]            Override path to local Zig cache directory
     \\  --global-cache-dir [path]     Override path to global Zig cache directory
+    \\  --cache-dependencies-locally  Cache dependencies locally in zig-deps
     \\  --zig-lib-dir [arg]           Override path to Zig lib directory
     \\  --build-runner [file]         Override path to build runner
     \\  --prominent-compile-errors    Buffer compile errors and display at end
@@ -4647,6 +4648,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
     var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena);
     var override_local_cache_dir: ?[]const u8 = try EnvVar.ZIG_LOCAL_CACHE_DIR.get(arena);
     var override_build_runner: ?[]const u8 = try EnvVar.ZIG_BUILD_RUNNER.get(arena);
+    var cache_dependencies_locally = false;
     var child_argv = std.ArrayList([]const u8).init(arena);
     var reference_trace: ?u32 = null;
     var debug_compile_errors = false;
@@ -4735,6 +4737,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                     i += 1;
                     override_global_cache_dir = args[i];
                     continue;
+                } else if (mem.eql(u8, arg, "--cache-dependencies-locally")) {
+                    cache_dependencies_locally = true;
+                    continue;
                 } else if (mem.eql(u8, arg, "-freference-trace")) {
                     reference_trace = 256;
                 } else if (mem.eql(u8, arg, "--fetch")) {
@@ -4860,6 +4865,15 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
     };
     defer global_cache_directory.handle.close();
 
+    var local_package_cache_root: ?Compilation.Directory = null;
+    if (cache_dependencies_locally) {
+        local_package_cache_root = .{
+            .handle = try fs.cwd().makeOpenPath("zig-deps", .{}),
+            .path = "zig-deps",
+        };
+    }
+    defer if (local_package_cache_root) |*lpc| lpc.handle.close();
+
     child_argv.items[argv_index_global_cache_dir] = global_cache_directory.path orelse cwd_path;
 
     var local_cache_directory: Compilation.Directory = l: {
@@ -4976,6 +4990,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                     .http_client = &http_client,
                     .thread_pool = &thread_pool,
                     .global_cache = global_cache_directory,
+                    .local_package_cache_root = local_package_cache_root,
                     .read_only = false,
                     .recursive = true,
                     .debug_hash = false,
@@ -6800,7 +6815,7 @@ const usage_fetch =
     \\  --save=[name]                 Add the fetched package to build.zig.zon as name
     \\  --save-exact                  Add the fetched package to build.zig.zon, storing the URL verbatim
     \\  --save-exact=[name]           Add the fetched package to build.zig.zon as name, storing the URL verbatim
-    \\
+    \\  --cache-dependencies-locally  Cache dependencies locally in zig-deps
 ;
 
 fn cmdFetch(
@@ -6813,6 +6828,7 @@ fn cmdFetch(
         EnvVar.ZIG_BTRFS_WORKAROUND.isSet();
     var opt_path_or_url: ?[]const u8 = null;
     var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena);
+    var cache_dependencies_locally = false;
     var debug_hash: bool = false;
     var save: union(enum) {
         no,
@@ -6837,6 +6853,8 @@ fn cmdFetch(
                     debug_hash = true;
                 } else if (mem.eql(u8, arg, "--save")) {
                     save = .{ .yes = null };
+                } else if (mem.eql(u8, arg, "--cache-dependencies-locally")) {
+                    cache_dependencies_locally = true;
                 } else if (mem.startsWith(u8, arg, "--save=")) {
                     save = .{ .yes = arg["--save=".len..] };
                 } else if (mem.eql(u8, arg, "--save-exact")) {
@@ -6879,10 +6897,20 @@ fn cmdFetch(
     };
     defer global_cache_directory.handle.close();
 
+    var local_package_cache_root: ?Compilation.Directory = null;
+    if (cache_dependencies_locally) {
+        local_package_cache_root = .{
+            .handle = try fs.cwd().makeOpenPath("zig-deps", .{}),
+            .path = "zig-deps",
+        };
+    }
+    defer if (local_package_cache_root) |*lpc| lpc.handle.close();
+
     var job_queue: Package.Fetch.JobQueue = .{
         .http_client = &http_client,
         .thread_pool = &thread_pool,
         .global_cache = global_cache_directory,
+        .local_package_cache_root = local_package_cache_root,
         .recursive = false,
         .read_only = false,
         .debug_hash = debug_hash,