From 2d03f3ac44807d50e1f62e9f1692fb787f8de0a2 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Mon, 18 Oct 2021 19:41:08 -0700 Subject: [PATCH] Locale improvements (#136) * Improve `locale` installation by prepopulating `/etc/locale.gen` This ensures that the list of valid locales is in-place when `debootstrap` starts configuring packages, such as `perl`. * Isolate build from host environment variables The build should not be influenced by things such as `LANG` set in the host; so we disable inheritance of `ENV` by manually calling `setenv()` every time. This unfortunately alters the signature of both the `do` block when calling `debootstrap()` and `chroot()`, but luckily Julia's dynamism makes it easy to work around. Co-authored-by: Dilum Aluthge --- .gitignore | 1 + linux/agent_linux.jl | 20 +++++++++++--------- linux/debian_minimal.jl | 2 +- linux/package_linux.jl | 24 ++++++++++++----------- src/build_img/alpine.jl | 9 ++++++--- src/build_img/debian.jl | 42 +++++++++++++++++++++++++++-------------- src/utils/chroot.jl | 4 +++- 7 files changed, 63 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index ccd0575..9164eca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ build/ temp/ tmp/ +.vscode/ diff --git a/linux/agent_linux.jl b/linux/agent_linux.jl index 6999f43..8ddf66c 100644 --- a/linux/agent_linux.jl +++ b/linux/agent_linux.jl @@ -19,7 +19,9 @@ packages = [ "wget", ] -artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs +artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs, chroot_ENV + root_chroot(args...) = chroot(args...; ENV=chroot_ENV, uid=0, gid=0) + @info("Installing buildkite-agent...") buildkite_install_cmd = """ echo 'deb https://apt.buildkite.com/buildkite-agent stable main' >> /etc/apt/sources.list && \\ @@ -27,10 +29,10 @@ artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do ro apt-get update && \\ DEBIAN_FRONTEND=noninteractive apt-get install -y buildkite-agent """ - chroot(rootfs, "bash", "-c", buildkite_install_cmd; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "which buildkite-agent"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "which -a buildkite-agent"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "buildkite-agent --help"; uid=0, gid=0) + root_chroot(rootfs, "bash", "-c", buildkite_install_cmd) + root_chroot(rootfs, "bash", "-c", "which buildkite-agent") + root_chroot(rootfs, "bash", "-c", "which -a buildkite-agent") + root_chroot(rootfs, "bash", "-c", "buildkite-agent --help") @info("Installing yq...") yq_install_cmd = """ @@ -40,10 +42,10 @@ artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do ro cd / && \\ rm -rfv /tmp-install-yq """ - chroot(rootfs, "bash", "-c", yq_install_cmd; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "which yq"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "which -a yq"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "yq --version"; uid=0, gid=0) + root_chroot(rootfs, "bash", "-c", yq_install_cmd) + root_chroot(rootfs, "bash", "-c", "which yq") + root_chroot(rootfs, "bash", "-c", "which -a yq") + root_chroot(rootfs, "bash", "-c", "yq --version") end upload_gha(tarball_path) diff --git a/linux/debian_minimal.jl b/linux/debian_minimal.jl index 3081fb7..b8bb3fc 100644 --- a/linux/debian_minimal.jl +++ b/linux/debian_minimal.jl @@ -6,7 +6,7 @@ archive = args.archive image = args.image packages = String[] -locale = false +locale = nothing artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages, locale) upload_gha(tarball_path) diff --git a/linux/package_linux.jl b/linux/package_linux.jl index cf221e1..07720b1 100644 --- a/linux/package_linux.jl +++ b/linux/package_linux.jl @@ -32,7 +32,9 @@ packages = [ "wget", ] -artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs +artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs, chroot_ENV + root_chroot(args...) = chroot(args...; ENV=chroot_ENV, uid=0, gid=0) + # Install GCC 9, specifically @info("Installing gcc-9") gcc_install_cmd = """ @@ -47,16 +49,16 @@ artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do ro ln -sf "\${tool}-9" "/usr/bin/\${tool}" done """ - chroot(rootfs, "bash", "-c", gcc_install_cmd; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "which gcc"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "which -a gcc"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "which g++"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "which -a g++"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "which gfortran"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "which -a gfortran"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "gcc --version"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "g++ --version"; uid=0, gid=0) - chroot(rootfs, "bash", "-c", "gfortran --version"; uid=0, gid=0) + root_chroot(rootfs, "bash", "-c", gcc_install_cmd) + root_chroot(rootfs, "bash", "-c", "which gcc") + root_chroot(rootfs, "bash", "-c", "which -a gcc") + root_chroot(rootfs, "bash", "-c", "which g++") + root_chroot(rootfs, "bash", "-c", "which -a g++") + root_chroot(rootfs, "bash", "-c", "which gfortran") + root_chroot(rootfs, "bash", "-c", "which -a gfortran") + root_chroot(rootfs, "bash", "-c", "gcc --version") + root_chroot(rootfs, "bash", "-c", "g++ --version") + root_chroot(rootfs, "bash", "-c", "gfortran --version") end upload_gha(tarball_path) diff --git a/src/build_img/alpine.jl b/src/build_img/alpine.jl index 810f9f8..a9f20b2 100644 --- a/src/build_img/alpine.jl +++ b/src/build_img/alpine.jl @@ -19,7 +19,10 @@ function alpine_bootstrap(f::Function, name::String; Pkg.Artifacts.download_verify_unpack(rootfs_url, nothing, rootfs; verbose=true) # Call user callback, if requested - f(rootfs) + chroot_ENV = Dict{String,String}( + "PATH" => "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + ) + f(rootfs, chroot_ENV) # Remove special `dev` files, take ownership, force symlinks to be relative, etc... rootfs_info = """ @@ -41,9 +44,9 @@ function alpine_bootstrap(f::Function, name::String; for pkg in filter(pkg -> pkg.repo == repo, packages) push!(apk_args, pkg.name) end - chroot(rootfs, apk_args...) + chroot(rootfs, apk_args...; ENV=chroot_ENV) end end end # If no user callback is provided, default to doing nothing -alpine_bootstrap(name::String; kwargs...) = alpine_bootstrap(p -> nothing, name; kwargs...) +alpine_bootstrap(name::String; kwargs...) = alpine_bootstrap((p, e) -> nothing, name; kwargs...) diff --git a/src/build_img/debian.jl b/src/build_img/debian.jl index c683f47..5a3a3a0 100644 --- a/src/build_img/debian.jl +++ b/src/build_img/debian.jl @@ -12,7 +12,7 @@ end function debootstrap(f::Function, arch::String, name::String; archive::Bool = true, force::Bool = false, - locale::Bool = true, + locale::Union{Nothing,String} = "en_US.UTF-8 UTF-8", packages::Vector{String} = String[], release::String = "buster", variant::String = "minbase") @@ -20,7 +20,7 @@ function debootstrap(f::Function, arch::String, name::String; error("Must install `debootstrap`!") end - if locale + if locale !== nothing if "locales" ∉ packages msg = string( "You have set the `locale` keyword argument to `true`. ", @@ -38,7 +38,25 @@ function debootstrap(f::Function, arch::String, name::String; error("Must install qemu-user-static and binfmt_misc!") end + chroot_ENV = Dict{String,String}( + "PATH" => "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + ) + + # If `locale` is set, pass that through as `LANG` + if locale !== nothing + chroot_ENV["LANG"] = first(split(locale)) + end + return create_rootfs(name; archive, force) do rootfs + # If `locale` is set, the first thing we do is to pre-populate `/etc/locales.gen` + if locale !== nothing + @info("Setting up locale", locale) + mkpath(joinpath(rootfs, "etc")) + open(joinpath(rootfs, "etc", "locale.gen"), "a") do io + println(io, locale) + end + end + @info("Running debootstrap", release, variant, packages) debootstrap_cmd = `sudo debootstrap` push!(debootstrap_cmd.exec, "--arch=$(debian_arch(arch))") @@ -51,17 +69,17 @@ function debootstrap(f::Function, arch::String, name::String; end push!(debootstrap_cmd.exec, "$(release)") push!(debootstrap_cmd.exec, "$(rootfs)") - run(debootstrap_cmd) + run(setenv(debootstrap_cmd, chroot_ENV)) # This is necessary on any 32-bit userspaces to work around the # following bad interaction between qemu, linux and openssl: # https://serverfault.com/questions/1045118/debootstrap-armhd-buster-unable-to-get-local-issuer-certificate if isfile(joinpath(rootfs, "usr", "bin", "c_rehash")) - chroot(rootfs, "/usr/bin/c_rehash"; uid=0, gid=0) + chroot(rootfs, "/usr/bin/c_rehash"; ENV=chroot_ENV, uid=0, gid=0) end # Call user callback, if requested - f(rootfs) + f(rootfs, chroot_ENV) # Remove special `dev` files, take ownership, force symlinks to be relative, etc... rootfs_info=""" @@ -84,19 +102,15 @@ function debootstrap(f::Function, arch::String, name::String; end end - # Set up the one true locale - if locale - @info("Setting up UTF-8 locale") - open(joinpath(rootfs, "etc", "locale.gen"), "a") do io - println(io, "en_US.UTF-8 UTF-8") - end - chroot(rootfs, "locale-gen") + # If we have locale support, ensure that `locale-gen` is run at least once. + if locale !== nothing + chroot(rootfs, "locale-gen"; ENV=chroot_ENV) end # Run `apt clean` - chroot(rootfs, "apt", "clean") + chroot(rootfs, "apt", "clean"; ENV=chroot_ENV) end end # If no user callback is provided, default to doing nothing -debootstrap(arch::String, name::String; kwargs...) = debootstrap(p -> nothing, arch, name; kwargs...) +debootstrap(arch::String, name::String; kwargs...) = debootstrap((p, e) -> nothing, arch, name; kwargs...) diff --git a/src/utils/chroot.jl b/src/utils/chroot.jl index 5333b1b..5536b7f 100644 --- a/src/utils/chroot.jl +++ b/src/utils/chroot.jl @@ -1,4 +1,6 @@ # Utility functions getuid() = ccall(:getuid, Cint, ()) getgid() = ccall(:getgid, Cint, ()) -chroot(rootfs, cmds...; uid=getuid(), gid=getgid()) = run(`sudo chroot --userspec=$(uid):$(gid) $(rootfs) $(cmds)`) +function chroot(rootfs, cmds...; ENV=copy(ENV), uid=getuid(), gid=getgid()) + run(setenv(`sudo chroot --userspec=$(uid):$(gid) $(rootfs) $(cmds)`, ENV)) +end