introduce package id and redo hash format again

Introduces the `id` field to `build.zig.zon`.

Together with name, this represents a globally unique package
identifier. This field should be initialized with a 16-bit random number
when the package is first created, and then *never change*. This allows
Zig to unambiguously detect when one package is an updated version of
another.

When forking a Zig project, this id should be regenerated with a new
random number if the upstream project is still maintained. Otherwise,
the fork is *hostile*, attempting to take control over the original
project's identity.

`0x0000` is invalid because it obviously means a random number wasn't
used.

`0xffff` is reserved to represent "naked" packages.

Tracking issue #14288

Additionally:

* Fix bad path in error messages regarding build.zig.zon file.
* Manifest validates that `name` and `version` field of build.zig.zon
  are maximum 32 bytes.
* Introduce error for root package to not switch to enum literal for
  name.
* Introduce error for root package to omit `id`.
* Update init template to generate `id`
* Update init template to populate `minimum_zig_version`.
* New package hash format changes:
  - name and version limited to 32 bytes via error rather than truncation
  - truncate sha256 to 192 bits rather than 40 bits
  - include the package id

This means that, given only the package hashes for a complete dependency
tree, it is possible to perform version selection and know the final
size on disk, without doing any fetching whatsoever. This prevents
wasted bandwidth since package versions not selected do not need to be
fetched.

Author: andrewrk

doc/build.zig.zon.md

@@ -10,7 +10,7 @@ build.zig.
 
 ### `name`
 
-String. Required.
+Enum literal. Required.
 
 This is the default name used by packages depending on this one. For example,
 when a user runs `zig fetch --save <url>`, this field is used as the key in the
@@ -20,12 +20,31 @@ will stick with this provided value.
 It is redundant to include "zig" in this name because it is already within the
 Zig package namespace.
 
+Must be a valid bare Zig identifier (don't `@` me), limited to 32 bytes.
+
+### `id`
+
+Together with name, this represents a globally unique package identifier. This
+field should be initialized with a 16-bit random number when the package is
+first created, and then *never change*. This allows Zig to unambiguously detect
+when one package is an updated version of another.
+
+When forking a Zig project, this id should be regenerated with a new random
+number if the upstream project is still maintained. Otherwise, the fork is
+*hostile*, attempting to take control over the original project's identity.
+
+`0x0000` is invalid because it obviously means a random number wasn't used.
+
+`0xffff` is reserved to represent "naked" packages.
+
 ### `version`
 
 String. Required.
 
 [semver](https://semver.org/)
 
+Limited to 32 bytes.
+
 ### `minimum_zig_version`
 
 String. Optional.

lib/init/build.zig

@@ -42,14 +42,14 @@ pub fn build(b: *std.Build) void {
     // Modules can depend on one another using the `std.Build.Module.addImport` function.
     // This is what allows Zig source code to use `@import("foo")` where 'foo' is not a
     // file path. In this case, we set up `exe_mod` to import `lib_mod`.
-    exe_mod.addImport("$_lib", lib_mod);
+    exe_mod.addImport("$n_lib", lib_mod);
 
     // Now, we will create a static library based on the module we created above.
     // This creates a `std.Build.Step.Compile`, which is the build step responsible
     // for actually invoking the compiler.
     const lib = b.addLibrary(.{
         .linkage = .static,
-        .name = "$",
+        .name = "$n",
         .root_module = lib_mod,
     });
 
@@ -61,7 +61,7 @@ pub fn build(b: *std.Build) void {
     // This creates another `std.Build.Step.Compile`, but this one builds an executable
     // rather than a static library.
     const exe = b.addExecutable(.{
-        .name = "$",
+        .name = "$n",
         .root_module = exe_mod,
     });
 

lib/init/build.zig.zon

@@ -6,12 +6,29 @@
     //
     // It is redundant to include "zig" in this name because it is already
     // within the Zig package namespace.
-    .name = "$",
+    .name = .$n,
 
     // This is a [Semantic Version](https://semver.org/).
     // In a future version of Zig it will be used for package deduplication.
     .version = "0.0.0",
 
+    // Together with name, this represents a globally unique package
+    // identifier. This field should be initialized with a 16-bit random number
+    // when the package is first created, and then *never change*. This allows
+    // unambiguous detection when one package is an updated version of another.
+    //
+    // When forking a Zig project, this id should be regenerated with a new
+    // random number if the upstream project is still maintained. Otherwise,
+    // the fork is *hostile*, attempting to take control over the original
+    // project's identity. Thus it is recommended to leave the comment on the
+    // following line intact, so that it shows up in code reviews that modify
+    // the field.
+    .id = $i, // Changing this has security and trust implications.
+
+    // Tracks the earliest Zig version that the package considers to be a
+    // supported use case.
+    .minimum_zig_version = "$v",
+
     // This field is optional.
     // This is currently advisory only; Zig does not yet do anything
     // with this value.

lib/init/src/main.zig

@@ -43,4 +43,4 @@ test "fuzz example" {
 const std = @import("std");
 
 /// This imports the separate module containing `root.zig`. Take a look in `build.zig` for details.
-const lib = @import("$_lib");
+const lib = @import("$n_lib");

src/Package.zig

@@ -10,9 +10,17 @@ pub const multihash_len = 1 + 1 + Hash.Algo.digest_length;
 pub const multihash_hex_digest_len = 2 * multihash_len;
 pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
 
+pub fn randomId() u16 {
+    return std.crypto.random.intRangeLessThan(u16, 0x0001, 0xffff);
+}
+
 /// A user-readable, file system safe hash that identifies an exact package
 /// snapshot, including file contents.
 ///
+/// The hash is not only to prevent collisions but must resist attacks where
+/// the adversary fully controls the contents being hashed. Thus, it contains
+/// a full SHA-256 digest.
+///
 /// This data structure can be used to store the legacy hash format too. Legacy
 /// hash format is scheduled to be removed after 0.14.0 is tagged.
 ///
@@ -26,7 +34,8 @@ pub const Hash = struct {
     pub const Algo = std.crypto.hash.sha2.Sha256;
     pub const Digest = [Algo.digest_length]u8;
 
-    pub const max_len = 32 + 1 + 32 + 1 + 12;
+    /// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh"
+    pub const max_len = 32 + 1 + 32 + 1 + (16 + 32 + 192) / 6;
 
     pub fn fromSlice(s: []const u8) Hash {
         assert(s.len <= max_len);
@@ -62,48 +71,35 @@ pub const Hash = struct {
         try std.testing.expect(h.isOld());
     }
 
-    /// Produces "$name-$semver-$sizedhash".
+    /// Produces "$name-$semver-$hashplus".
     /// * name is the name field from build.zig.zon, truncated at 32 bytes and must
     ///   be a valid zig identifier
     /// * semver is the version field from build.zig.zon, truncated at 32 bytes
-    /// * sizedhash is the following 9-byte array, base64 encoded using -_ to make
+    /// * hashplus is the following 39-byte array, base64 encoded using -_ to make
     ///   it filesystem safe:
-    ///   - (4 bytes) LE u32 total decompressed size in bytes
-    ///   - (5 bytes) truncated SHA-256 of hashed files of the package
+    ///   - (2 bytes) LE u16 Package ID
+    ///   - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated
+    ///   - (24 bytes) truncated SHA-256 digest of hashed files of the package
     ///
-    /// example: "nasm-2.16.1-2-BWdcABvF_jM1"
-    pub fn init(digest: Digest, name: []const u8, ver: []const u8, size: u32) Hash {
+    /// example: "nasm-2.16.1-3-AAD_ZlwACpGU-c3QXp_yNyn07Q5U9Rq-Cb1ur2G1"
+    pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u16, size: u32) Hash {
+        assert(name.len <= 32);
+        assert(ver.len <= 32);
         var result: Hash = undefined;
         var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes);
-        buf.appendSliceAssumeCapacity(name[0..@min(name.len, 32)]);
+        buf.appendSliceAssumeCapacity(name);
         buf.appendAssumeCapacity('-');
-        buf.appendSliceAssumeCapacity(ver[0..@min(ver.len, 32)]);
+        buf.appendSliceAssumeCapacity(ver);
         buf.appendAssumeCapacity('-');
-        var sizedhash: [9]u8 = undefined;
-        std.mem.writeInt(u32, sizedhash[0..4], size, .little);
-        sizedhash[4..].* = digest[0..5].*;
-        _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(12), &sizedhash);
+        var hashplus: [30]u8 = undefined;
+        std.mem.writeInt(u16, hashplus[0..2], id, .little);
+        std.mem.writeInt(u32, hashplus[2..6], size, .little);
+        hashplus[6..].* = digest[0..24].*;
+        _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(40), &hashplus);
         @memset(buf.unusedCapacitySlice(), 0);
         return result;
     }
 
-    /// Produces "$hashiname-N-$sizedhash". For packages that lack "build.zig.zon" metadata.
-    /// * hashiname is [5..][0..24] bytes of the SHA-256, urlsafe-base64-encoded, for a total of 32 bytes encoded
-    /// * the semver section is replaced with a hardcoded N which stands for
-    ///   "naked". It acts as a version number so that any future updates to the
-    ///   hash format can tell this hash format apart. Note that "N" is an
-    ///   invalid semver.
-    /// * sizedhash is the same as in `init`.
-    ///
-    /// The hash is broken up this way so that "sizedhash" can be calculated
-    /// exactly the same way in both cases, and so that "name" and "hashiname" can
-    /// be used interchangeably in both cases.
-    pub fn initNaked(digest: Digest, size: u32) Hash {
-        var name: [32]u8 = undefined;
-        _ = std.base64.url_safe_no_pad.Encoder.encode(&name, digest[5..][0..24]);
-        return init(digest, &name, "N", size);
-    }
-
     /// Produces a unique hash based on the path provided. The result should
     /// not be user-visible.
     pub fn initPath(sub_path: []const u8, is_global: bool) Hash {
@@ -144,7 +140,7 @@ pub const MultihashFunction = enum(u16) {
 
 pub const multihash_function: MultihashFunction = switch (Hash.Algo) {
     std.crypto.hash.sha2.Sha256 => .@"sha2-256",
-    else => @compileError("unreachable"),
+    else => unreachable,
 };
 
 pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest {

src/Package/Fetch.zig

@@ -586,9 +586,11 @@ pub fn computedPackageHash(f: *const Fetch) Package.Hash {
     if (f.manifest) |man| {
         var version_buffer: [32]u8 = undefined;
         const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer;
-        return .init(f.computed_hash.digest, man.name, version, saturated_size);
+        return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size);
     }
-    return .initNaked(f.computed_hash.digest, saturated_size);
+    // In the future build.zig.zon fields will be added to allow overriding these values
+    // for naked tarballs.
+    return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size);
 }
 
 /// `computeHash` gets a free check for the existence of `build.zig`, but when
@@ -645,11 +647,13 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
 
     f.manifest = try Manifest.parse(arena, ast.*, .{
         .allow_missing_paths_field = f.allow_missing_paths_field,
+        .allow_missing_id = f.allow_missing_paths_field,
+        .allow_name_string = f.allow_missing_paths_field,
     });
     const manifest = &f.manifest.?;
 
     if (manifest.errors.len > 0) {
-        const src_path = try eb.printString("{}{s}", .{ pkg_root, Manifest.basename });
+        const src_path = try eb.printString("{}" ++ fs.path.sep_str ++ "{s}", .{ pkg_root, Manifest.basename });
         try manifest.copyErrorsIntoBundle(ast.*, src_path, eb);
         return error.FetchFailed;
     }

src/Package/Manifest.zig

@@ -36,6 +36,7 @@ pub const ErrorMessage = struct {
 };
 
 name: []const u8,
+id: u16,
 version: std.SemanticVersion,
 version_node: Ast.Node.Index,
 dependencies: std.StringArrayHashMapUnmanaged(Dependency),
@@ -50,6 +51,8 @@ pub const ParseOptions = struct {
     allow_missing_paths_field: bool = false,
     /// Deprecated, to be removed after 0.14.0 is tagged.
     allow_name_string: bool = true,
+    /// Deprecated, to be removed after 0.14.0 is tagged.
+    allow_missing_id: bool = true,
 };
 
 pub const Error = Allocator.Error;
@@ -70,13 +73,15 @@ pub fn parse(gpa: Allocator, ast: Ast, options: ParseOptions) Error!Manifest {
         .errors = .{},
 
         .name = undefined,
+        .id = 0,
         .version = undefined,
         .version_node = 0,
         .dependencies = .{},
         .dependencies_node = 0,
         .paths = .{},
         .allow_missing_paths_field = options.allow_missing_paths_field,
         .allow_name_string = options.allow_name_string,
+        .allow_missing_id = options.allow_missing_id,
         .minimum_zig_version = null,
         .buf = .{},
     };
@@ -92,6 +97,7 @@ pub fn parse(gpa: Allocator, ast: Ast, options: ParseOptions) Error!Manifest {
 
     return .{
         .name = p.name,
+        .id = p.id,
         .version = p.version,
         .version_node = p.version_node,
         .dependencies = try p.dependencies.clone(p.arena),
@@ -143,13 +149,15 @@ const Parse = struct {
     errors: std.ArrayListUnmanaged(ErrorMessage),
 
     name: []const u8,
+    id: u16,
     version: std.SemanticVersion,
     version_node: Ast.Node.Index,
     dependencies: std.StringArrayHashMapUnmanaged(Dependency),
     dependencies_node: Ast.Node.Index,
     paths: std.StringArrayHashMapUnmanaged(void),
     allow_missing_paths_field: bool,
     allow_name_string: bool,
+    allow_missing_id: bool,
     minimum_zig_version: ?std.SemanticVersion,
 
     const InnerError = error{ ParseFailure, OutOfMemory };
@@ -167,6 +175,7 @@ const Parse = struct {
         var have_name = false;
         var have_version = false;
         var have_included_paths = false;
+        var have_id = false;
 
         for (struct_init.ast.fields) |field_init| {
             const name_token = ast.firstToken(field_init) - 2;
@@ -183,6 +192,9 @@ const Parse = struct {
             } else if (mem.eql(u8, field_name, "name")) {
                 p.name = try parseName(p, field_init);
                 have_name = true;
+            } else if (mem.eql(u8, field_name, "id")) {
+                p.id = try parseId(p, field_init);
+                have_id = true;
             } else if (mem.eql(u8, field_name, "version")) {
                 p.version_node = field_init;
                 const version_text = try parseString(p, field_init);
@@ -206,6 +218,12 @@ const Parse = struct {
             }
         }
 
+        if (!have_id and !p.allow_missing_id) {
+            try appendError(p, main_token, "missing top-level 'id' field; suggested value: 0x{x}", .{
+                Package.randomId(),
+            });
+        }
+
         if (!have_name) {
             try appendError(p, main_token, "missing top-level 'name' field", .{});
         }
@@ -359,6 +377,33 @@ const Parse = struct {
         }
     }
 
+    fn parseId(p: *Parse, node: Ast.Node.Index) !u16 {
+        const ast = p.ast;
+        const node_tags = ast.nodes.items(.tag);
+        const main_tokens = ast.nodes.items(.main_token);
+        const main_token = main_tokens[node];
+        if (node_tags[node] != .number_literal) {
+            return fail(p, main_token, "expected integer literal", .{});
+        }
+        const token_bytes = ast.tokenSlice(main_token);
+        const parsed = std.zig.parseNumberLiteral(token_bytes);
+        const n = switch (parsed) {
+            .int => |n| n,
+            .big_int, .float => return fail(p, main_token, "expected u16 integer literal, found {s}", .{
+                @tagName(parsed),
+            }),
+            .failure => |err| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}),
+        };
+        const casted = std.math.cast(u16, n) orelse
+            return fail(p, main_token, "integer value {d} does not fit into u16", .{n});
+        switch (casted) {
+            0x0000, 0xffff => return fail(p, main_token, "id value 0x{x} reserved; use 0x{x} instead", .{
+                casted, Package.randomId(),
+            }),
+            else => return casted,
+        }
+    }
+
     fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 {
         const ast = p.ast;
         const node_tags = ast.nodes.items(.tag);
@@ -371,7 +416,7 @@ const Parse = struct {
                 return fail(p, main_token, "name must be a valid bare zig identifier (hint: switch from string to enum literal)", .{});
 
             if (name.len > max_name_len)
-                return fail(p, main_token, "name '{s}' exceeds max length of {d}", .{
+                return fail(p, main_token, "name '{}' exceeds max length of {d}", .{
                     std.zig.fmtId(name), max_name_len,
                 });
 
@@ -386,7 +431,7 @@ const Parse = struct {
             return fail(p, main_token, "name must be a valid bare zig identifier", .{});
 
         if (ident_name.len > max_name_len)
-            return fail(p, main_token, "name '{s}' exceeds max length of {d}", .{
+            return fail(p, main_token, "name '{}' exceeds max length of {d}", .{
                 std.zig.fmtId(ident_name), max_name_len,
             });