diff --git a/src/Package.zig b/src/Package.zig index 2e1dd4e14f1e..bf567d70d20d 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -625,7 +625,80 @@ const HashedFile = struct { } }; -fn computePackageHash( +pub fn computePackageHashExcludingDirectories( + thread_pool: *ThreadPool, + pkg_dir: fs.IterableDir, + excluded_directories: []const []const u8, +) ![Manifest.Hash.digest_length]u8 { + const gpa = thread_pool.allocator; + + // We'll use an arena allocator for the path name strings since they all + // need to be in memory for sorting. + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + // Collect all files, recursively, then sort. + var all_files = std.ArrayList(*HashedFile).init(gpa); + defer all_files.deinit(); + + var walker = try pkg_dir.walk(gpa); + defer walker.deinit(); + + { + // The final hash will be a hash of each file hashed independently. This + // allows hashing in parallel. + var wait_group: WaitGroup = .{}; + defer wait_group.wait(); + + loop: while (try walker.next()) |entry| { + switch (entry.kind) { + .Directory => { + for (excluded_directories) |dir_name| { + if (mem.eql(u8, entry.basename, dir_name)) { + var item = walker.stack.pop(); + if (walker.stack.items.len != 0) { + item.iter.dir.close(); + } + continue :loop; + } + } + continue :loop; + }, + .File => {}, + else => return error.IllegalFileTypeInPackage, + } + const hashed_file = try arena.create(HashedFile); + const fs_path = try arena.dupe(u8, entry.path); + hashed_file.* = .{ + .fs_path = fs_path, + .normalized_path = try normalizePath(arena, fs_path), + .hash = undefined, // to be populated by the worker + .failure = undefined, // to be populated by the worker + }; + wait_group.start(); + try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); + + try all_files.append(hashed_file); + } + } + + std.sort.sort(*HashedFile, all_files.items, {}, HashedFile.lessThan); + + var hasher = Manifest.Hash.init(.{}); + var any_failures = false; + for (all_files.items) |hashed_file| { + hashed_file.failure catch |err| { + any_failures = true; + std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) }); + }; + hasher.update(&hashed_file.hash); + } + if (any_failures) return error.PackageHashUnavailable; + return hasher.finalResult(); +} + +pub fn computePackageHash( thread_pool: *ThreadPool, pkg_dir: fs.IterableDir, ) ![Manifest.Hash.digest_length]u8 { diff --git a/src/main.zig b/src/main.zig index ffe5a9c8ea9a..a8c7cf3b4e5d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -16,6 +16,7 @@ const tracy = @import("tracy.zig"); const Compilation = @import("Compilation.zig"); const link = @import("link.zig"); const Package = @import("Package.zig"); +const Manifest = @import("Manifest.zig"); const build_options = @import("build_options"); const introspect = @import("introspect.zig"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; @@ -304,6 +305,8 @@ pub fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi return cmdFmt(gpa, arena, cmd_args); } else if (mem.eql(u8, cmd, "objcopy")) { return @import("objcopy.zig").cmdObjCopy(gpa, arena, cmd_args); + } else if (mem.eql(u8, cmd, "pkg")) { + return cmdPkg(gpa, arena, cmd_args); } else if (mem.eql(u8, cmd, "libc")) { return cmdLibC(gpa, cmd_args); } else if (mem.eql(u8, cmd, "init-exe")) { @@ -4179,6 +4182,81 @@ pub fn cmdInit( } } +pub const usage_pkg = + \\Usage: zig pkg [command] [options] + \\ + \\ Runs a package command + \\ + \\Commands: + \\ hash Calculates the package hash of the current directory. + \\ + \\Options: + \\ -h --help Print this help and exit. + \\ + \\Sub-options for [hash]: + \\ --allow-directory + \\ +; + +pub fn cmdPkg(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { + _ = arena; + if (args.len == 0) fatal("Expected at least one argument.\n", .{}); + const command_arg = args[0]; + + if (mem.eql(u8, command_arg, "-h") or mem.eql(u8, command_arg, "--help")) { + const stdout = io.getStdOut().writer(); + try stdout.writeAll(usage_pkg); + return cleanExit(); + } + + if (!mem.eql(u8, args[0], "hash")) fatal("Invalid command: {s}\n", .{command_arg}); + + const cwd = std.fs.cwd(); + + dir_test: { + if (args.len > 1 and mem.eql(u8, args[1], "--allow-directory")) break :dir_test; + try if (cwd.access(Package.build_zig_basename, .{})) |_| break :dir_test else |err| switch (err) { + error.FileNotFound => {}, + else => |e| e, + }; + try if (cwd.access(Manifest.basename, .{})) |_| break :dir_test else |err| switch (err) { + error.FileNotFound => {}, + else => |e| e, + }; + break :dir_test fatal("Could not find either build.zig or build.zig.zon in this directory.\n Use --allow-directory to override this check.\n", .{}); + } + + const hash = blk: { + const cwd_absolute_path = try cwd.realpathAlloc(gpa, "."); + defer gpa.free(cwd_absolute_path); + + // computePackageHash will close the directory after completion + var cwd_copy = try fs.openIterableDirAbsolute(cwd_absolute_path, .{}); + errdefer cwd_copy.dir.close(); + + var thread_pool: ThreadPool = undefined; + try thread_pool.init(.{ .allocator = gpa }); + defer thread_pool.deinit(); + + // workaround for missing inclusion/exclusion support -> #14311. + const excluded_directories: []const []const u8 = &.{ + "zig-out", + "zig-cache", + ".git", + }; + break :blk try Package.computePackageHashExcludingDirectories( + &thread_pool, + .{ .dir = cwd_copy.dir }, + excluded_directories, + ); + }; + + const std_out = std.io.getStdOut(); + const digest = Manifest.hexDigest(hash); + try std_out.writeAll(digest[0..]); + try std_out.writeAll("\n"); +} + pub const usage_build = \\Usage: zig build [steps] [options] \\