Skip to content

Commit

Permalink
Use statically-linked bsdtar on all platforms (#804)
Browse files Browse the repository at this point in the history 
* [tar] Switch to statically linked binaries

* chore: don't use bsdtar for extract yet

* chore: add dzbarsky integrity hashes

* Upgrade to 3.7.2.bcr.2 (#806)

* chore: replace URLs with aspect-build fork

---------

Co-authored-by: David Zbarsky <[email protected]>
  • Loading branch information
@alexeagle @dzbarsky
alexeagle and dzbarsky committed Mar 29, 2024
1 parent 4109b3c commit 036d1fd
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 135 deletions.
7 changes: 1 addition & 6 deletions MODULE.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,7 @@ register_toolchains(
"@yq_toolchains//:all",
"@coreutils_toolchains//:all",
"@expand_template_toolchains//:all",
# Expand bsd_tar_toolchains
"@bsd_tar_toolchains//:linux_amd64_toolchain",
"@bsd_tar_toolchains//:linux_arm64_toolchain",
"@bsd_tar_toolchains//:windows_amd64_toolchain",
# host toolchain must be last, as it's only suitable as a fallback on macos
"@bsd_tar_toolchains//:host_toolchain",
"@bsd_tar_toolchains//:all",
)

# To allow /tools to be built from source
Expand Down
181 changes: 52 additions & 129 deletions lib/private/tar_toolchain.bzl
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,18 @@
"Provide access to a BSD tar"

load(":repo_utils.bzl", "repo_utils")

BSDTAR_PLATFORMS = {
"darwin_amd64": struct(
compatible_with = [
"@platforms//os:osx",
"@platforms//cpu:x86_64",
],
),
"darwin_arm64": struct(
compatible_with = [
"@platforms//os:osx",
"@platforms//cpu:aarch64",
],
),
"linux_amd64": struct(
compatible_with = [
"@platforms//os:linux",
Expand All @@ -22,145 +32,58 @@ BSDTAR_PLATFORMS = {
"@platforms//cpu:x86_64",
],
),
# WARNING: host toolchain should always come last to make it a fallback toolchain.
"host": struct(
# loaded by the macro
compatible_with = "HOST_CONSTRAINTS",
),
}

WINDOWS_DEPS = (
"e06f10043b1b148eb38ad06cff678af05beade0bdd2edd8735a198c521fa3993",
"https://github.com/libarchive/libarchive/releases/download/v3.7.2/libarchive-v3.7.2-amd64.zip",
)

# note, using Ubuntu Focal packages as they link with older glibc versions.
# Ubuntu Jammy packages will fail on ubuntu 20.02 with
# bsdtar: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.33' not found
# bsdtar: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found
#
# TODO: this is only a partial listing of the transitive deps of libarchive-tools
# so we expect a bunch of compress modes are broken, for example.

LINUX_LIB_DEPS = {
"linux_arm64": [
(
"6d18525e248e84b8a4ee39a226fd1195ca9b9d0d5a1c7909ae4f997d46378848",
"http://ports.ubuntu.com/pool/main/n/nettle/libnettle7_3.5.1+really3.5.1-2ubuntu0.2_arm64.deb",
),
(
"aa5e31d05a9d6bde8093137bd1c82b5a20a5f470bd5109642014f895c20f323a",
"http://ports.ubuntu.com/pool/main/liba/libarchive/libarchive13_3.4.0-2ubuntu1_arm64.deb",
),
(
"6d089f878507b536d8ca51b1ad80a80706a1dd7dbbcce7600800d3f9f98be2ab",
"http://ports.ubuntu.com/pool/main/liba/libarchive/libarchive-tools_3.2.1-2~ubuntu16.04.1_arm64.deb",
),
(
"6242892cb032859044ddfcfbe61bac5678a95c585d8fff4525acaf45512e3d39",
"http://ports.ubuntu.com/pool/main/libx/libxml2/libxml2_2.9.10+dfsg-5_arm64.deb",
),
(
"6302e309ab002af30ddfa0d68de26c68f7c034ed2f45b1d97a712bff1a03999a",
"http://ports.ubuntu.com/pool/main/i/icu/libicu66_66.1-2ubuntu2_arm64.deb",
),
],
"linux_amd64": [
# https://packages.ubuntu.com/focal/amd64/libarchive-tools/download
(
"12a19878d34b407e6f4893d3b26b7758a26c5534a066d76184c8b764b2df1652",
"http://security.ubuntu.com/ubuntu/pool/universe/liba/libarchive/libarchive-tools_3.4.0-2ubuntu1.2_amd64.deb",
),
# https://packages.ubuntu.com/focal/amd64/libarchive13/download
(
"8ba7507f61bb3ea8da488702ec0badcbfb726d36ea6886e3421ac59082aaf2d1",
"http://security.ubuntu.com/ubuntu/pool/main/liba/libarchive/libarchive13_3.4.0-2ubuntu1.2_amd64.deb",
),
# https://packages.ubuntu.com/focal/amd64/libnettle7/download
(
"3496aed83407fde71e0dc5988b28e8fd7f07a2f27fcf3e0f214c7cd86667eecd",
"http://security.ubuntu.com/ubuntu/pool/main/n/nettle/libnettle7_3.5.1+really3.5.1-2ubuntu0.2_amd64.deb",
),
# https://packages.ubuntu.com/focal/amd64/libxml2/download
(
"cef3871873e5e7e7f1d01afec4d35f15504a9d8a0afbe56f57037e92f7f8e850",
"http://security.ubuntu.com/ubuntu/pool/main/libx/libxml2/libxml2_2.9.10+dfsg-5ubuntu0.20.04.7_amd64.deb",
),
# https://packages.ubuntu.com/focal/amd64/libicu66/download
(
"00d0de456134668f41bd9ea308a076bc0a6a805180445af8a37209d433f41efe",
"http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu66_66.1-2ubuntu2.1_amd64.deb",
),
],
BSDTAR_PREBUILT = {
"darwin_amd64": (
"https://github.com/aspect-build/bsdtar-prebuilt/releases/download/v3.7.2/tar_darwin_amd64",
"a3bd0e7be92bcddcb70341f1efc48c29ef99b3ad57349b433e9a3182b68cb0c1",
),
"darwin_arm64": (
"https://github.com/aspect-build/bsdtar-prebuilt/releases/download/v3.7.2/tar_darwin_arm64",
"63ee769e2d870d1ed3542e292c919dc8a3934544d17b2de34213c18c41c5437f",
),
"linux_amd64": (
"https://github.com/aspect-build/bsdtar-prebuilt/releases/download/v3.7.2/tar_linux_amd64",
"d40582e64aace892e2f9588045edc5e67023ca3371cd575e7043b0c2a08205b4",
),
"linux_arm64": (
"https://github.com/aspect-build/bsdtar-prebuilt/releases/download/v3.7.2/tar_linux_arm64",
"e2527be38499e94e021c7c02476b4cff8083313d38c85bdf46fc5751d31d32aa",
),
"windows_amd64": (
"https://github.com/libarchive/libarchive/releases/download/v3.7.2/libarchive-v3.7.2-amd64.zip",
"e06f10043b1b148eb38ad06cff678af05beade0bdd2edd8735a198c521fa3993",
),
}

def _find_usable_system_tar(rctx, tar_name):
tar = rctx.which(tar_name)
if not tar:
fail("tar not found on PATH, and we don't handle this case yet")

# Run tar --version and see if we are satisfied to use it
tar_version = rctx.execute([tar, "--version"]).stdout.strip()

# TODO: also check if it's really ancient or compiled without gzip support or something?
# TODO: document how users could fetch the source and compile it themselves
if tar_version.find("bsdtar") >= 0:
return tar

fail("tar isn't a BSD tar")

def _bsdtar_binary_repo(rctx):
tar_name = "tar.exe" if repo_utils.is_windows(rctx) else "tar"
build_header = """\
# @generated by @aspect_bazel_lib//lib/private:tar_toolchain.bzl
load("@aspect_bazel_lib//lib/private:tar_toolchain.bzl", "tar_toolchain")
package(default_visibility = ["//visibility:public"])
"""

# On MacOS, the system `tar` binary on the PATH should already work
if rctx.attr.platform == "host":
tar = _find_usable_system_tar(rctx, tar_name)
output = rctx.path(tar_name)
rctx.symlink(tar, output)
rctx.file("BUILD.bazel", build_header + """tar_toolchain(name = "bsdtar_toolchain", binary = "tar")""")
return

if repo_utils.is_windows(rctx):
(url, sha256) = BSDTAR_PREBUILT[rctx.attr.platform]
if rctx.attr.platform.startswith("windows"):
rctx.download_and_extract(
url = WINDOWS_DEPS[1],
url = url,
type = "zip",
sha256 = WINDOWS_DEPS[0],
sha256 = sha256,
)
binary = "libarchive/bin/bsdtar.exe"
else:
rctx.download(
url = url,
output = "tar",
executable = True,
sha256 = sha256,
)
rctx.file("BUILD.bazel", build_header + """tar_toolchain(name = "bsdtar_toolchain", binary = "libarchive/bin/bsdtar.exe")""")
return
binary = "tar"

# Other platforms, we have more work to do.
libs_dir = "usr/lib/x86_64-linux-gnu" if rctx.attr.platform.endswith("amd64") else "usr/lib/aarch64-linux-gnu"
rctx.file("BUILD.bazel", """\
# @generated by @aspect_bazel_lib//lib/private:tar_toolchain.bzl
for lib in LINUX_LIB_DEPS[rctx.attr.platform]:
rctx.download_and_extract(
url = lib[1],
type = "deb",
sha256 = lib[0],
)
rctx.extract("data.tar.xz")
load("@aspect_bazel_lib//lib/private:tar_toolchain.bzl", "tar_toolchain")
rctx.file("bsdtar.sh", """#!/usr/bin/env bash
readonly wksp="$(dirname "${{BASH_SOURCE[0]}}")"
LD_LIBRARY_PATH=$wksp/{libs_dir} exec $wksp/usr/bin/bsdtar $@
""".format(name = rctx.name, libs_dir = libs_dir))
package(default_visibility = ["//visibility:public"])
rctx.file("BUILD.bazel", build_header + """\
tar_toolchain(
name = "bsdtar_toolchain",
files = glob(["{libs}/*.so.*"]) + ["usr/bin/bsdtar"],
binary = "bsdtar.sh",
visibility = ["//visibility:public"],
)
""".format(libs = libs_dir, name = rctx.name))
tar_toolchain(name = "bsdtar_toolchain", binary = "{}")
""".format(binary))

bsdtar_binary_repo = repository_rule(
implementation = _bsdtar_binary_repo,
Expand Down

0 comments on commit 036d1fd

Please sign in to comment.