Skip to content

Add support for local copy of dependency trees #20150

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Add support for local copy of dependency trees #20150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions lib/compiler/build_runner.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,8 @@ pub fn main() !void {
builder.verbose_link = true;
} else if (mem.eql(u8, arg, "--verbose-air")) {
builder.verbose_air = true;
} else if (mem.eql(u8, arg, "--cache-dependencies-locally")) {
builder.cache_dependencies_locally = true;
} else if (mem.eql(u8, arg, "--verbose-llvm-ir")) {
builder.verbose_llvm_ir = "-";
} else if (mem.startsWith(u8, arg, "--verbose-llvm-ir=")) {
Expand Down Expand Up @@ -1184,6 +1186,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void {
\\ --build-file [file] Override path to build.zig
\\ --cache-dir [path] Override path to local Zig cache directory
\\ --global-cache-dir [path] Override path to global Zig cache directory
\\ --cache-dependencies-locally Cache dependencies locally in zig-deps
\\ --zig-lib-dir [arg] Override path to Zig lib directory
\\ --build-runner [file] Override path to build runner
\\ --seed [integer] For shuffling dependency traversal order (default: random)
Expand Down
3 changes: 3 additions & 0 deletions lib/std/Build.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ verbose: bool,
verbose_link: bool,
verbose_cc: bool,
verbose_air: bool,
cache_dependencies_locally: bool,
verbose_llvm_ir: ?[]const u8,
verbose_llvm_bc: ?[]const u8,
verbose_cimport: bool,
Expand Down Expand Up @@ -254,6 +255,7 @@ pub fn create(
.graph = graph,
.build_root = build_root,
.cache_root = cache_root,
.cache_dependencies_locally = false,
.verbose = false,
.verbose_link = false,
.verbose_cc = false,
Expand Down Expand Up @@ -380,6 +382,7 @@ fn createChildOnly(
.installed_files = ArrayList(InstalledFile).init(allocator),
.build_root = build_root,
.cache_root = parent.cache_root,
.cache_dependencies_locally = parent.cache_dependencies_locally,
.zig_lib_dir = parent.zig_lib_dir,
.debug_log_scopes = parent.debug_log_scopes,
.debug_compile_errors = parent.debug_compile_errors,
Expand Down
2 changes: 1 addition & 1 deletion lib/std/Build/Step/Compile.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1380,7 +1380,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
if (b.debug_compile_errors) {
try zig_args.append("--debug-compile-errors");
}

if (b.cache_dependencies_locally) try zig_args.append("--cache-dependencies-locally");
if (b.verbose_cimport) try zig_args.append("--verbose-cimport");
if (b.verbose_air) try zig_args.append("--verbose-air");
if (b.verbose_llvm_ir) |path| try zig_args.append(b.fmt("--verbose-llvm-ir={s}", .{path}));
Expand Down
183 changes: 143 additions & 40 deletions src/Package/Fetch.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,18 @@
//! Represents one independent job whose responsibility is to:
//!
//! 1. Check the global zig package cache to see if the hash already exists.
//! If so, load, parse, and validate the build.zig.zon file therein, and
//! goto step 8. Likewise if the location is a relative path, treat this
//! the same as a cache hit. Otherwise, proceed.
//! 1. if local zig packages is enabled:
//! a. Check the local zig package directory to see if the hash already exists.
//! If so, load, parse, and validate the build.zig.zon file therein, and
//! goto step 8. Likewise if the location is a relative path, treat this
//! the same as a cache hit. Otherwise, proceed.
//! b. Check the global zig package cache to see if the hash already exists.
//! If so, load, parse, and validate the build.zig.zon file therein, copy
//! it to the local zig package directory and goto step 8. Otherwise, proceed.
//! if local zig packages is disabled:
//! Check the global zig package cache to see if the hash already exists.
//! If so, load, parse, and validate the build.zig.zon file therein, and
//! goto step 8. Likewise if the location is a relative path, treat this
//! the same as a cache hit. Otherwise, proceed.
//! 2. Fetch and unpack a URL into a temporary directory.
//! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed
//! for the file to be missing, in which case this fetched package is considered
Expand All @@ -18,7 +27,8 @@
//! directory. If the hash already exists, delete the temporary directory and
//! leave the zig package cache directory untouched as it may be in use by the
//! system. This is done even if the hash is invalid, in case the package with
//! the different hash is used in the future.
//! the different hash is used in the future. If local packages is enabled, copy
//! the package locally.
//! 7. Validate the computed hash against the expected hash. If invalid,
//! this job is done.
//! 8. Spawn a new fetch job for each dependency in the manifest file. Use
Expand Down Expand Up @@ -97,6 +107,7 @@ pub const JobQueue = struct {
thread_pool: *ThreadPool,
wait_group: WaitGroup = .{},
global_cache: Cache.Directory,
local_package_cache_root: ?Cache.Directory,
/// If true then, no fetching occurs, and:
/// * The `global_cache` directory is assumed to be the direct parent
/// directory of on-disk packages rather than having the "p/" directory
Expand Down Expand Up @@ -299,7 +310,7 @@ pub fn run(f: *Fetch) RunError!void {
const arena = f.arena.allocator();
const gpa = f.arena.child_allocator;
const cache_root = f.job_queue.global_cache;

const local_package_cache = f.job_queue.local_package_cache_root;
try eb.init(gpa);

// Check the global zig package cache to see if the hash already exists. If
Expand Down Expand Up @@ -372,41 +383,102 @@ pub fn run(f: *Fetch) RunError!void {
const prefixed_pkg_sub_path = "p" ++ s ++ expected_hash;
const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
assert(f.lazy_status != .unavailable);
f.package_root = .{
.root_dir = cache_root,
.sub_path = try arena.dupe(u8, pkg_sub_path),
};
try loadManifest(f, f.package_root);
try checkBuildFileExistence(f);
if (!f.job_queue.recursive) return;
return queueJobsForDeps(f);
} else |err| switch (err) {
error.FileNotFound => {
switch (f.lazy_status) {
.eager => {},
.available => if (!f.job_queue.unlazy_set.contains(expected_hash)) {
f.lazy_status = .unavailable;
return;
},
.unavailable => unreachable,
var check_cache_root = local_package_cache == null;
if (local_package_cache) |lpc| {
if (lpc.handle.access(&expected_hash, .{})) {
f.package_root = .{
.root_dir = lpc,
.sub_path = try arena.dupe(u8, &expected_hash),
};
try loadManifest(f, f.package_root);
try checkBuildFileExistence(f);
if (!f.job_queue.recursive) return;
return queueJobsForDeps(f);
} else |err| switch (err) {
error.FileNotFound => {
check_cache_root = true;
},
else => |e| {
try eb.addRootErrorMessage(.{
.msg = try eb.printString("unable to open local package cache directory '{}{s}': {s}", .{
lpc, &expected_hash, @errorName(e),
}),
});
return error.FetchFailed;
},
}
}
if (check_cache_root) {
if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
assert(f.lazy_status != .unavailable);
if (local_package_cache) |lpc| {
var dst_handle = lpc.handle.makeOpenPath(&expected_hash, .{}) catch |err| {
try eb.addRootErrorMessage(.{
.msg = try eb.printString("unable to open local package cache directory '{}{s}': {s}", .{
lpc, &expected_hash, @errorName(err),
}),
});
return error.FetchFailed;
};
defer dst_handle.close();
var src_handle = cache_root.handle.openDir(pkg_sub_path, .{}) catch |err| {
try eb.addRootErrorMessage(.{
.msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{
cache_root, pkg_sub_path, @errorName(err),
}),
});
return error.FetchFailed;
};
defer src_handle.close();
f.recursiveDirectoryCopy(src_handle, dst_handle) catch |err| {
const src = try cache_root.join(arena, &.{pkg_sub_path});
const dest = try lpc.join(arena, &.{&expected_hash});
try eb.addRootErrorMessage(.{ .msg = try eb.printString(
"sunable to copy into local package directory '{s}' from package cache directory '{s}': {s}",
.{ dest, src, @errorName(err) },
) });
return error.FetchFailed;
};
f.package_root = .{
.root_dir = lpc,
.sub_path = try arena.dupe(u8, &expected_hash),
};
} else {
f.package_root = .{
.root_dir = cache_root,
.sub_path = try arena.dupe(u8, pkg_sub_path),
};
}
if (f.job_queue.read_only) return f.fail(
f.name_tok,
try eb.printString("package not found at '{}{s}'", .{
cache_root, pkg_sub_path,
}),
);
},
else => |e| {
try eb.addRootErrorMessage(.{
.msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{
cache_root, pkg_sub_path, @errorName(e),
}),
});
return error.FetchFailed;
},
try loadManifest(f, f.package_root);
try checkBuildFileExistence(f);
if (!f.job_queue.recursive) return;
return queueJobsForDeps(f);
} else |err| switch (err) {
error.FileNotFound => {
switch (f.lazy_status) {
.eager => {},
.available => if (!f.job_queue.unlazy_set.contains(expected_hash)) {
f.lazy_status = .unavailable;
return;
},
.unavailable => unreachable,
}
if (f.job_queue.read_only) return f.fail(
f.name_tok,
try eb.printString("package not found at '{}{s}'", .{
cache_root, pkg_sub_path,
}),
);
},
else => |e| {
try eb.addRootErrorMessage(.{
.msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{
cache_root, pkg_sub_path, @errorName(e),
}),
});
return error.FetchFailed;
},
}
}
} else if (f.job_queue.read_only) {
try eb.addRootErrorMessage(.{
Expand Down Expand Up @@ -444,6 +516,7 @@ fn runResource(
const eb = &f.error_bundle;
const s = fs.path.sep_str;
const cache_root = f.job_queue.global_cache;
const local_package_cache = f.job_queue.local_package_cache_root;
const rand_int = std.crypto.random.int(u64);
const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);

Expand Down Expand Up @@ -526,6 +599,36 @@ fn runResource(
) });
return error.FetchFailed;
};
if (local_package_cache) |lpc| {
var dst_handle = lpc.handle.makeOpenPath(&Manifest.hexDigest(f.actual_hash), .{}) catch |err| {
try eb.addRootErrorMessage(.{
.msg = try eb.printString("unable to open local package cache directory '{}{s}': {s}", .{
lpc, &Manifest.hexDigest(f.actual_hash), @errorName(err),
}),
});
return error.FetchFailed;
};
defer dst_handle.close();
var src_handle = cache_root.handle.openDir(f.package_root.sub_path, .{}) catch |err| {
try eb.addRootErrorMessage(.{
.msg = try eb.printString("unable to open global package cache directory '{}{s}': {s}", .{
cache_root, f.package_root.sub_path, @errorName(err),
}),
});
return error.FetchFailed;
};
defer src_handle.close();
f.recursiveDirectoryCopy(src_handle, dst_handle) catch |err| {
const src = try cache_root.join(arena, &.{f.package_root.sub_path});
const dest = try lpc.join(arena, &.{&Manifest.hexDigest(f.actual_hash)});
try eb.addRootErrorMessage(.{ .msg = try eb.printString(
"unable to copy into local package directory '{s}' from package cache directory '{s}': {s}",
.{ dest, src, @errorName(err) },
) });
return error.FetchFailed;
};
}

// Remove temporary directory root if not already renamed to global cache.
if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) {
cache_root.handle.deleteDir(tmp_dir_sub_path) catch {};
Expand Down
30 changes: 29 additions & 1 deletion src/main.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4632,6 +4632,7 @@ const usage_build =
\\ --build-file [file] Override path to build.zig
\\ --cache-dir [path] Override path to local Zig cache directory
\\ --global-cache-dir [path] Override path to global Zig cache directory
\\ --cache-dependencies-locally Cache dependencies locally in zig-deps
\\ --zig-lib-dir [arg] Override path to Zig lib directory
\\ --build-runner [file] Override path to build runner
\\ --prominent-compile-errors Buffer compile errors and display at end
Expand All @@ -4647,6 +4648,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena);
var override_local_cache_dir: ?[]const u8 = try EnvVar.ZIG_LOCAL_CACHE_DIR.get(arena);
var override_build_runner: ?[]const u8 = try EnvVar.ZIG_BUILD_RUNNER.get(arena);
var cache_dependencies_locally = false;
var child_argv = std.ArrayList([]const u8).init(arena);
var reference_trace: ?u32 = null;
var debug_compile_errors = false;
Expand Down Expand Up @@ -4735,6 +4737,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
i += 1;
override_global_cache_dir = args[i];
continue;
} else if (mem.eql(u8, arg, "--cache-dependencies-locally")) {
cache_dependencies_locally = true;
continue;
} else if (mem.eql(u8, arg, "-freference-trace")) {
reference_trace = 256;
} else if (mem.eql(u8, arg, "--fetch")) {
Expand Down Expand Up @@ -4860,6 +4865,15 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
};
defer global_cache_directory.handle.close();

var local_package_cache_root: ?Compilation.Directory = null;
if (cache_dependencies_locally) {
local_package_cache_root = .{
.handle = try fs.cwd().makeOpenPath("zig-deps", .{}),
.path = "zig-deps",
};
}
defer if (local_package_cache_root) |*lpc| lpc.handle.close();

child_argv.items[argv_index_global_cache_dir] = global_cache_directory.path orelse cwd_path;

var local_cache_directory: Compilation.Directory = l: {
Expand Down Expand Up @@ -4976,6 +4990,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
.http_client = &http_client,
.thread_pool = &thread_pool,
.global_cache = global_cache_directory,
.local_package_cache_root = local_package_cache_root,
.read_only = false,
.recursive = true,
.debug_hash = false,
Expand Down Expand Up @@ -6800,7 +6815,7 @@ const usage_fetch =
\\ --save=[name] Add the fetched package to build.zig.zon as name
\\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim
\\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim
\\
\\ --cache-dependencies-locally Cache dependencies locally in zig-deps
;

fn cmdFetch(
Expand All @@ -6813,6 +6828,7 @@ fn cmdFetch(
EnvVar.ZIG_BTRFS_WORKAROUND.isSet();
var opt_path_or_url: ?[]const u8 = null;
var override_global_cache_dir: ?[]const u8 = try EnvVar.ZIG_GLOBAL_CACHE_DIR.get(arena);
var cache_dependencies_locally = false;
var debug_hash: bool = false;
var save: union(enum) {
no,
Expand All @@ -6837,6 +6853,8 @@ fn cmdFetch(
debug_hash = true;
} else if (mem.eql(u8, arg, "--save")) {
save = .{ .yes = null };
} else if (mem.eql(u8, arg, "--cache-dependencies-locally")) {
cache_dependencies_locally = true;
} else if (mem.startsWith(u8, arg, "--save=")) {
save = .{ .yes = arg["--save=".len..] };
} else if (mem.eql(u8, arg, "--save-exact")) {
Expand Down Expand Up @@ -6879,10 +6897,20 @@ fn cmdFetch(
};
defer global_cache_directory.handle.close();

var local_package_cache_root: ?Compilation.Directory = null;
if (cache_dependencies_locally) {
local_package_cache_root = .{
.handle = try fs.cwd().makeOpenPath("zig-deps", .{}),
.path = "zig-deps",
};
}
defer if (local_package_cache_root) |*lpc| lpc.handle.close();

var job_queue: Package.Fetch.JobQueue = .{
.http_client = &http_client,
.thread_pool = &thread_pool,
.global_cache = global_cache_directory,
.local_package_cache_root = local_package_cache_root,
.recursive = false,
.read_only = false,
.debug_hash = debug_hash,
Expand Down