From 472bf9b12210240c01c703171d9906b67386f32a Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Mon, 9 Oct 2023 15:57:52 -0700 Subject: [PATCH] feat: tar includes runfiles (#595) * feat: tar includes runfiles * chore: try to fix red circleci * fix: tracked down problem * chore: document tar#srcs supports runfiles * chore: add comment about logic for trimming manifest suffix * chore: missed a replacement spot * chore: give up on the listing test for now --- docs/tar.md | 2 +- lib/private/BUILD.bazel | 13 +++++---- lib/private/tar.bzl | 55 ++++++++++++++++++++++++++++++----- lib/private/tar_toolchain.bzl | 1 - lib/tests/tar/BUILD.bazel | 36 +++++++++++++++++++++++ lib/tests/tar/asserts.bzl | 1 + lib/tests/tar/cat_src_file.sh | 15 ++++++++++ 7 files changed, 108 insertions(+), 15 deletions(-) create mode 100755 lib/tests/tar/cat_src_file.sh diff --git a/docs/tar.md b/docs/tar.md index e33da2457..d7f94ba5b 100644 --- a/docs/tar.md +++ b/docs/tar.md @@ -61,7 +61,7 @@ Rule that executes BSD `tar`. Most users should use the [`tar`](#tar) macro, rat | mode | A mode indicator from the following list, copied from the tar manpage:

- create: Create a new archive containing the specified items. - append: Like create, but new entries are appended to the archive. Note that this only works on uncompressed archives stored in regular files. The -f option is required. - list: List archive contents to stdout. - update: Like append, but new entries are added only if they have a modification date newer than the corresponding entry in the archive. Note that this only works on uncompressed archives stored in regular files. The -f option is required. - extract: Extract to disk from the archive. If a file with the same name appears more than once in the archive, each copy will be extracted, with later copies overwriting (replacing) earlier copies. | String | optional | "create" | | mtree | An mtree specification file | Label | required | | | out | Resulting tar file to write. If absent, [name].tar is written. | Label | optional | | -| srcs | Files and directories that are placed into the tar | List of labels | required | | +| srcs | Files, directories, or other targets whose default outputs are placed into the tar.

If any of the srcs are binaries with runfiles, those are copied into the resulting tar as well. | List of labels | required | | diff --git a/lib/private/BUILD.bazel b/lib/private/BUILD.bazel index 115aea46b..faa70c77e 100644 --- a/lib/private/BUILD.bazel +++ b/lib/private/BUILD.bazel @@ -267,6 +267,13 @@ bzl_library( deps = [":repo_utils"], ) +bzl_library( + name = "tar", + srcs = ["tar.bzl"], + visibility = ["//lib:__subpackages__"], + deps = ["@aspect_bazel_lib//lib:paths"], +) + bzl_library( name = "copy_common", srcs = ["copy_common.bzl"], @@ -320,9 +327,3 @@ bzl_library( srcs = ["strings.bzl"], visibility = ["//lib:__subpackages__"], ) - -bzl_library( - name = "tar", - srcs = ["tar.bzl"], - visibility = ["//lib:__subpackages__"], -) diff --git a/lib/private/tar.bzl b/lib/private/tar.bzl index bc3c160ab..f3e9e2038 100644 --- a/lib/private/tar.bzl +++ b/lib/private/tar.bzl @@ -1,10 +1,17 @@ "Implementation of tar rule" + +load("@aspect_bazel_lib//lib:paths.bzl", "to_rlocation_path") + _tar_attrs = { "args": attr.string_list( doc = "Additional flags permitted by BSD tar; see the man page.", ), "srcs": attr.label_list( - doc = "Files and directories that are placed into the tar", + doc = """\ + Files, directories, or other targets whose default outputs are placed into the tar. + + If any of the srcs are binaries with runfiles, those are copied into the resulting tar as well. + """, mandatory = True, allow_files = True, ), @@ -67,6 +74,23 @@ def _add_compress_options(compress, args): if compress == "zstd": args.add("--zstd") +def _runfile_path(ctx, file, runfiles_dir): + return "/".join([runfiles_dir, to_rlocation_path(ctx, file)]) + +def _calculate_runfiles_dir(default_info): + manifest = default_info.files_to_run.runfiles_manifest + + # Newer versions of Bazel put the manifest besides the runfiles with the suffix .runfiles_manifest. + # For example, the runfiles directory is named my_binary.runfiles then the manifest is beside the + # runfiles directory and named my_binary.runfiles_manifest + # Older versions of Bazel put the manifest file named MANIFEST in the runfiles directory + # See similar logic: + # https://github.com/aspect-build/rules_js/blob/c50bd3f797c501fb229cf9ab58e0e4fc11464a2f/js/private/bash.bzl#L63 + if manifest.short_path.endswith("_manifest") or manifest.short_path.endswith("/MANIFEST"): + # Trim last 9 characters, as that's the length in both cases + return manifest.short_path[:-9] + fail("manifest path {} seems malformed".format(manifest.short_path)) + def _tar_impl(ctx): bsdtar = ctx.toolchains["@aspect_bazel_lib//lib:tar_toolchain_type"] inputs = ctx.files.srcs[:] @@ -89,7 +113,10 @@ def _tar_impl(ctx): ctx.actions.run( executable = bsdtar.tarinfo.binary, - inputs = depset(direct = inputs, transitive = [bsdtar.default.files]), + inputs = depset(direct = inputs, transitive = [bsdtar.default.files] + [ + src[DefaultInfo].default_runfiles.files + for src in ctx.attr.srcs + ]), outputs = [out], arguments = [args], mnemonic = "Tar", @@ -99,17 +126,17 @@ def _tar_impl(ctx): def _default_mtree_line(file): # Functions passed to map_each cannot take optional arguments. - return _mtree_line(file) + return _mtree_line(file.short_path, file.path, "dir" if file.is_directory else "file") -def _mtree_line(file, uid = "0", gid = "0", time = "1672560000", mode = "0755"): +def _mtree_line(file, content, type, uid = "0", gid = "0", time = "1672560000", mode = "0755"): return " ".join([ - file.short_path, + file, "uid=" + uid, "gid=" + gid, "time=" + time, "mode=" + mode, - "type=" + ("dir" if file.is_directory else "file"), - "content=" + file.path, + "type=" + type, + "content=" + content, ]) def _mtree_impl(ctx): @@ -118,6 +145,20 @@ def _mtree_impl(ctx): content = ctx.actions.args() content.set_param_file_format("multiline") content.add_all(ctx.files.srcs, map_each = _default_mtree_line) + + for s in ctx.attr.srcs: + default_info = s[DefaultInfo] + if not default_info.files_to_run.runfiles_manifest: + continue + + runfiles_dir = _calculate_runfiles_dir(default_info) + for file in depset(transitive = [s.default_runfiles.files]).to_list(): + destination = _runfile_path(ctx, file, runfiles_dir) + content.add("{} uid=0 gid=0 mode=0755 time=1672560000 type=file content={}".format( + destination, + file.path, + )) + ctx.actions.write(out, content = content) return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out])) diff --git a/lib/private/tar_toolchain.bzl b/lib/private/tar_toolchain.bzl index 6ab88cf50..fa4edb2fd 100644 --- a/lib/private/tar_toolchain.bzl +++ b/lib/private/tar_toolchain.bzl @@ -9,7 +9,6 @@ BSDTAR_PLATFORMS = { "@platforms//cpu:x86_64", ], ), - # TODO(alexeagle): download from somewhere "linux_arm64": struct( compatible_with = [ "@platforms//os:linux", diff --git a/lib/tests/tar/BUILD.bazel b/lib/tests/tar/BUILD.bazel index 0a3ecab31..c4981c051 100644 --- a/lib/tests/tar/BUILD.bazel +++ b/lib/tests/tar/BUILD.bazel @@ -1,6 +1,7 @@ load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:testing.bzl", "assert_archive_contains") load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") load("@bazel_skylib//rules:write_file.bzl", "write_file") load(":asserts.bzl", "assert_tar_listing") @@ -152,3 +153,38 @@ bzl_library( "@bazel_skylib//rules:write_file", ], ) + +# Case 6: Runfiles +sh_binary( + name = "cat_src_file", + srcs = ["cat_src_file.sh"], + data = ["src_file"], + deps = ["@bazel_tools//tools/bash/runfiles"], +) + +tar( + name = "tar_runfiles", + srcs = [":cat_src_file"], +) + +genrule( + name = "run_program_with_runfiles", + srcs = [":tar_runfiles"], + outs = ["cat_src_file_output"], + cmd = """\ + export DIR=$$(mktemp -d) + $(BSDTAR_BIN) --extract --file $(execpath :tar_runfiles) --directory $$DIR + ( + cd $$DIR + ./lib/tests/tar/cat_src_file + ) > $@ + """, + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], +) + +diff_test( + name = "test_runfiles", + timeout = "short", + file1 = "src_file", + file2 = "cat_src_file_output", +) diff --git a/lib/tests/tar/asserts.bzl b/lib/tests/tar/asserts.bzl index c0d9e6fff..a3ce57e9f 100644 --- a/lib/tests/tar/asserts.bzl +++ b/lib/tests/tar/asserts.bzl @@ -26,4 +26,5 @@ def assert_tar_listing(name, actual, expected): name = name, file1 = actual_listing, file2 = expected_listing, + timeout = "short", ) diff --git a/lib/tests/tar/cat_src_file.sh b/lib/tests/tar/cat_src_file.sh new file mode 100755 index 000000000..ce6c37328 --- /dev/null +++ b/lib/tests/tar/cat_src_file.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# A trivial program that only works when runfiles are resolved at runtime. + +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- + +cat $(rlocation aspect_bazel_lib/lib/tests/tar/src_file)