Locale improvements (#136)

* Improve `locale` installation by prepopulating `/etc/locale.gen` This ensures that the list of valid locales is in-place when `debootstrap` starts configuring packages, such as `perl`. * Isolate build from host environment variables The build should not be influenced by things such as `LANG` set in the host; so we disable inheritance of `ENV` by manually calling `setenv()` every time. This unfortunately alters the signature of both the `do` block when calling `debootstrap()` and `chroot()`, but luckily Julia's dynamism makes it easy to work around. Co-authored-by: Dilum Aluthge <[email protected]>
JuliaCI · Oct 19, 2021 · 2d03f3a · 2d03f3a
1 parent 670bedf
commit 2d03f3a
Show file tree

Hide file tree

Showing 7 changed files with 63 additions and 39 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 build/
 temp/
 tmp/
+.vscode/
diff --git a/linux/agent_linux.jl b/linux/agent_linux.jl
@@ -19,18 +19,20 @@ packages = [
     "wget",
 ]
 
-artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs
+artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs, chroot_ENV
+    root_chroot(args...) = chroot(args...; ENV=chroot_ENV, uid=0, gid=0)
+
     @info("Installing buildkite-agent...")
     buildkite_install_cmd = """
     echo 'deb https://apt.buildkite.com/buildkite-agent stable main' >> /etc/apt/sources.list && \\
     curl -sfL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x32A37959C2FA5C3C99EFBC32A79206696452D198" | apt-key add - && \\
     apt-get update && \\
     DEBIAN_FRONTEND=noninteractive apt-get install -y buildkite-agent
     """
-    chroot(rootfs, "bash", "-c", buildkite_install_cmd; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "which buildkite-agent"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "which -a buildkite-agent"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "buildkite-agent --help"; uid=0, gid=0)
+    root_chroot(rootfs, "bash", "-c", buildkite_install_cmd)
+    root_chroot(rootfs, "bash", "-c", "which buildkite-agent")
+    root_chroot(rootfs, "bash", "-c", "which -a buildkite-agent")
+    root_chroot(rootfs, "bash", "-c", "buildkite-agent --help")
 
     @info("Installing yq...")
     yq_install_cmd = """
@@ -40,10 +42,10 @@ artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do ro
     cd / && \\
     rm -rfv /tmp-install-yq
     """
-    chroot(rootfs, "bash", "-c", yq_install_cmd; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "which yq"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "which -a yq"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "yq --version"; uid=0, gid=0)
+    root_chroot(rootfs, "bash", "-c", yq_install_cmd)
+    root_chroot(rootfs, "bash", "-c", "which yq")
+    root_chroot(rootfs, "bash", "-c", "which -a yq")
+    root_chroot(rootfs, "bash", "-c", "yq --version")
 end
 
 upload_gha(tarball_path)

diff --git a/linux/debian_minimal.jl b/linux/debian_minimal.jl
@@ -6,7 +6,7 @@ archive      = args.archive
 image        = args.image
 
 packages = String[]
-locale = false
+locale = nothing
 
 artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages, locale)
 upload_gha(tarball_path)

diff --git a/linux/package_linux.jl b/linux/package_linux.jl
@@ -32,7 +32,9 @@ packages = [
     "wget",
 ]
 
-artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs
+artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do rootfs, chroot_ENV
+    root_chroot(args...) = chroot(args...; ENV=chroot_ENV, uid=0, gid=0)
+
     # Install GCC 9, specifically
     @info("Installing gcc-9")
     gcc_install_cmd = """
@@ -47,16 +49,16 @@ artifact_hash, tarball_path, = debootstrap(arch, image; archive, packages) do ro
         ln -sf "\${tool}-9" "/usr/bin/\${tool}"
     done
     """
-    chroot(rootfs, "bash", "-c", gcc_install_cmd; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "which gcc"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "which -a gcc"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "which g++"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "which -a g++"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "which gfortran"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "which -a gfortran"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "gcc --version"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "g++ --version"; uid=0, gid=0)
-    chroot(rootfs, "bash", "-c", "gfortran --version"; uid=0, gid=0)
+    root_chroot(rootfs, "bash", "-c", gcc_install_cmd)
+    root_chroot(rootfs, "bash", "-c", "which gcc")
+    root_chroot(rootfs, "bash", "-c", "which -a gcc")
+    root_chroot(rootfs, "bash", "-c", "which g++")
+    root_chroot(rootfs, "bash", "-c", "which -a g++")
+    root_chroot(rootfs, "bash", "-c", "which gfortran")
+    root_chroot(rootfs, "bash", "-c", "which -a gfortran")
+    root_chroot(rootfs, "bash", "-c", "gcc --version")
+    root_chroot(rootfs, "bash", "-c", "g++ --version")
+    root_chroot(rootfs, "bash", "-c", "gfortran --version")
 end
 
 upload_gha(tarball_path)

diff --git a/src/build_img/alpine.jl b/src/build_img/alpine.jl
@@ -19,7 +19,10 @@ function alpine_bootstrap(f::Function, name::String;
         Pkg.Artifacts.download_verify_unpack(rootfs_url, nothing, rootfs; verbose=true)
 
         # Call user callback, if requested
-        f(rootfs)
+        chroot_ENV = Dict{String,String}(
+            "PATH" => "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+        )
+        f(rootfs, chroot_ENV)
 
         # Remove special `dev` files, take ownership, force symlinks to be relative, etc...
         rootfs_info = """
@@ -41,9 +44,9 @@ function alpine_bootstrap(f::Function, name::String;
             for pkg in filter(pkg -> pkg.repo == repo, packages)
                 push!(apk_args, pkg.name)
             end
-            chroot(rootfs, apk_args...)
+            chroot(rootfs, apk_args...; ENV=chroot_ENV)
         end
     end
 end
 # If no user callback is provided, default to doing nothing
-alpine_bootstrap(name::String; kwargs...) = alpine_bootstrap(p -> nothing, name; kwargs...)
+alpine_bootstrap(name::String; kwargs...) = alpine_bootstrap((p, e) -> nothing, name; kwargs...)
diff --git a/src/build_img/debian.jl b/src/build_img/debian.jl
@@ -12,15 +12,15 @@ end
 function debootstrap(f::Function, arch::String, name::String;
                      archive::Bool = true,
                      force::Bool = false,
-                     locale::Bool = true,
+                     locale::Union{Nothing,String} = "en_US.UTF-8 UTF-8",
                      packages::Vector{String} = String[],
                      release::String = "buster",
                      variant::String = "minbase")
     if Sys.which("debootstrap") === nothing
         error("Must install `debootstrap`!")
     end
 
-    if locale
+    if locale !== nothing
         if "locales" ∉ packages
             msg = string(
                 "You have set the `locale` keyword argument to `true`. ",
@@ -38,7 +38,25 @@ function debootstrap(f::Function, arch::String, name::String;
         error("Must install qemu-user-static and binfmt_misc!")
     end
 
+    chroot_ENV = Dict{String,String}(
+        "PATH" => "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+    )
+
+    # If `locale` is set, pass that through as `LANG`
+    if locale !== nothing
+        chroot_ENV["LANG"] = first(split(locale))
+    end
+
     return create_rootfs(name; archive, force) do rootfs
+        # If `locale` is set, the first thing we do is to pre-populate `/etc/locales.gen`
+        if locale !== nothing
+            @info("Setting up locale", locale)
+            mkpath(joinpath(rootfs, "etc"))
+            open(joinpath(rootfs, "etc", "locale.gen"), "a") do io
+                println(io, locale)
+            end
+        end
+
         @info("Running debootstrap", release, variant, packages)
         debootstrap_cmd = `sudo debootstrap`
         push!(debootstrap_cmd.exec, "--arch=$(debian_arch(arch))")
@@ -51,17 +69,17 @@ function debootstrap(f::Function, arch::String, name::String;
         end
         push!(debootstrap_cmd.exec, "$(release)")
         push!(debootstrap_cmd.exec, "$(rootfs)")
-        run(debootstrap_cmd)
+        run(setenv(debootstrap_cmd, chroot_ENV))
 
         # This is necessary on any 32-bit userspaces to work around the
         # following bad interaction between qemu, linux and openssl:
         # https://serverfault.com/questions/1045118/debootstrap-armhd-buster-unable-to-get-local-issuer-certificate
         if isfile(joinpath(rootfs, "usr", "bin", "c_rehash"))
-            chroot(rootfs, "/usr/bin/c_rehash"; uid=0, gid=0)
+            chroot(rootfs, "/usr/bin/c_rehash"; ENV=chroot_ENV, uid=0, gid=0)
         end
 
         # Call user callback, if requested
-        f(rootfs)
+        f(rootfs, chroot_ENV)
 
         # Remove special `dev` files, take ownership, force symlinks to be relative, etc...
         rootfs_info="""
@@ -84,19 +102,15 @@ function debootstrap(f::Function, arch::String, name::String;
             end
         end
 
-        # Set up the one true locale
-        if locale
-            @info("Setting up UTF-8 locale")
-            open(joinpath(rootfs, "etc", "locale.gen"), "a") do io
-                println(io, "en_US.UTF-8 UTF-8")
-            end
-            chroot(rootfs, "locale-gen")
+        # If we have locale support, ensure that `locale-gen` is run at least once.
+        if locale !== nothing
+            chroot(rootfs, "locale-gen"; ENV=chroot_ENV)
         end
 
         # Run `apt clean`
-        chroot(rootfs, "apt", "clean")
+        chroot(rootfs, "apt", "clean"; ENV=chroot_ENV)
     end
 end
 
 # If no user callback is provided, default to doing nothing
-debootstrap(arch::String, name::String; kwargs...) = debootstrap(p -> nothing, arch, name; kwargs...)
+debootstrap(arch::String, name::String; kwargs...) = debootstrap((p, e) -> nothing, arch, name; kwargs...)
diff --git a/src/utils/chroot.jl b/src/utils/chroot.jl
@@ -1,4 +1,6 @@
 # Utility functions
 getuid() = ccall(:getuid, Cint, ())
 getgid() = ccall(:getgid, Cint, ())
-chroot(rootfs, cmds...; uid=getuid(), gid=getgid()) = run(`sudo chroot --userspec=$(uid):$(gid) $(rootfs) $(cmds)`)
+function chroot(rootfs, cmds...; ENV=copy(ENV), uid=getuid(), gid=getgid())
+    run(setenv(`sudo chroot --userspec=$(uid):$(gid) $(rootfs) $(cmds)`, ENV))
+end