diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0eb2f61..d8da85d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,8 +83,9 @@ jobs: - run: sudo apt-get update - run: sudo apt-get -y install binfmt-support debootstrap qemu-user-static - run: update-binfmts --display - - run: julia --color=yes --project=. -e 'using Pkg; Pkg.instantiate()' - - run: julia --color=yes --project=. -e 'using Pkg; Pkg.precompile()' + - run: julia --color=yes --project=. -e 'using Pkg; @time Pkg.instantiate()' + - run: julia --color=yes --project=. -e 'using Pkg; @time Pkg.precompile()' + - run: julia --color=yes --project=. -e '@time using RootfsUtils' - run: | IMAGE_NAME=$(echo ${{ matrix.image }} | cut -d. -f1) IMAGE_ARCH=$(echo ${{ matrix.image }} | cut -d. -f2) diff --git a/Manifest.toml b/Manifest.toml index 03f448d..9b9629d 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -124,6 +124,10 @@ uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +[[Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + [[TextWrap]] git-tree-sha1 = "9250ef9b01b66667380cf3275b3f7488d0e25faf" uuid = "b718987f-49a8-5099-9789-dcd902bef87d" diff --git a/Project.toml b/Project.toml index edde3ee..0fed24a 100644 --- a/Project.toml +++ b/Project.toml @@ -1,9 +1,14 @@ +name = "RootfsUtils" +uuid = "ea0daf2f-8d67-45cc-b2d9-fd898f843992" + [deps] ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" Sandbox = "9307e30f-c43e-9ca7-d17c-c2dc59df670d" Scratch = "6c6a2e73-6563-6170-7368-637461726353" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533" [compat] diff --git a/images/agent_linux.jl b/images/agent_linux.jl index 861be24..90acbf3 100644 --- a/images/agent_linux.jl +++ b/images/agent_linux.jl @@ -2,8 +2,9 @@ ## to run inside of. Most CI steps will be run within a different image ## nested inside of this one. -include(joinpath(dirname(@__DIR__), "rootfs_utils.jl")) -arch, = parse_args(ARGS) +using RootfsUtils + +arch, = parse_build_args(ARGS) image = "$(splitext(basename(@__FILE__))[1]).$(arch)" # Build debian-based image with the following extra packages: diff --git a/images/llvm_passes.jl b/images/llvm_passes.jl index 00a12fe..eb55ebc 100644 --- a/images/llvm_passes.jl +++ b/images/llvm_passes.jl @@ -1,7 +1,8 @@ ## This rootfs includes enough of a host toolchain to build the LLVM passes (such as `analyzegc`). -include(joinpath(dirname(@__DIR__), "rootfs_utils.jl")) -arch, = parse_args(ARGS) +using RootfsUtils + +arch, = parse_build_args(ARGS) image = "$(splitext(basename(@__FILE__))[1]).$(arch)" # Build debian-based image with the following extra packages: diff --git a/images/package_linux.jl b/images/package_linux.jl index 9440593..8995381 100644 --- a/images/package_linux.jl +++ b/images/package_linux.jl @@ -1,8 +1,9 @@ ## This rootfs includes everything that must be installed to build Julia ## within a debian-based environment with GCC 9. -include(joinpath(dirname(@__DIR__), "rootfs_utils.jl")) -arch, = parse_args(ARGS) +using RootfsUtils + +arch, = parse_build_args(ARGS) image = "$(splitext(basename(@__FILE__))[1]).$(arch)" # Build debian-based image with the following extra packages: diff --git a/images/package_linux_qemu.jl b/images/package_linux_qemu.jl index a52b991..e4801eb 100644 --- a/images/package_linux_qemu.jl +++ b/images/package_linux_qemu.jl @@ -1,8 +1,9 @@ ## This rootfs includes everything that must be installed to build Julia ## within a debian-based environment with GCC 9. -include(joinpath(dirname(@__DIR__), "rootfs_utils.jl")) -arch, = parse_args(ARGS) +using RootfsUtils + +arch, = parse_build_args(ARGS) image = "$(splitext(basename(@__FILE__))[1]).$(arch)" # Build debian-based image with the following extra packages: diff --git a/images/package_musl.jl b/images/package_musl.jl index 5c132a4..1bd4da8 100644 --- a/images/package_musl.jl +++ b/images/package_musl.jl @@ -1,8 +1,9 @@ ## This rootfs includes everything that must be installed to build Julia ## within an alpine-based environment with GCC 9. -include(joinpath(dirname(@__DIR__), "rootfs_utils.jl")) -arch, = parse_args(ARGS) +using RootfsUtils + +arch, = parse_build_args(ARGS) image = "$(splitext(basename(@__FILE__))[1]).$(arch)" # Build alpine-based image with the following extra packages: diff --git a/images/tester_linux.jl b/images/tester_linux.jl index 6fa4cab..2682112 100644 --- a/images/tester_linux.jl +++ b/images/tester_linux.jl @@ -1,7 +1,8 @@ ## This rootfs does not include the compiler toolchain. -include(joinpath(dirname(@__DIR__), "rootfs_utils.jl")) -arch, = parse_args(ARGS) +using RootfsUtils + +arch, = parse_build_args(ARGS) image = "$(splitext(basename(@__FILE__))[1]).$(arch)" # Build debian-based image with the following extra packages: diff --git a/images/tester_musl.jl b/images/tester_musl.jl index 114bc8f..780f9a8 100644 --- a/images/tester_musl.jl +++ b/images/tester_musl.jl @@ -1,7 +1,8 @@ ## This rootfs does not include the compiler toolchain. -include(joinpath(dirname(@__DIR__), "rootfs_utils.jl")) -arch, = parse_args(ARGS) +using RootfsUtils + +arch, = parse_build_args(ARGS) image = "$(splitext(basename(@__FILE__))[1]).$(arch)" # Build alpine-based image with the following extra packages: diff --git a/src/RootfsUtils.jl b/src/RootfsUtils.jl new file mode 100644 index 0000000..587e120 --- /dev/null +++ b/src/RootfsUtils.jl @@ -0,0 +1,32 @@ +module RootfsUtils + +using ArgParse: ArgParse +using Base.BinaryPlatforms +using Dates +using Pkg +using Pkg.Artifacts +using SHA +using Scratch +using Test +using ghr_jll + +export AlpinePackage +export alpine_bootstrap +export chroot +export debootstrap +export parse_build_args +export test_sandbox +export upload_rootfs_image_github_actions + +include("types.jl") + +include("build/alpine.jl") +include("build/args.jl") +include("build/common.jl") +include("build/debian.jl") +include("build/utils.jl") + +include("run/args.jl") +include("run/test.jl") + +end # module diff --git a/src/build/alpine.jl b/src/build/alpine.jl new file mode 100644 index 0000000..4e6b957 --- /dev/null +++ b/src/build/alpine.jl @@ -0,0 +1,44 @@ +function repository_arg(repo) + if startswith(repo, "https://") + return "--repository=$(repo)" + end + return "--repository=http://dl-cdn.alpinelinux.org/alpine/$(repo)/main" +end + +function alpine_bootstrap(f::Function, name::String; release::VersionNumber=v"3.13.5", variant="minirootfs", + packages::Vector{AlpinePackage}=AlpinePackage[], force::Bool=false) + return create_rootfs(name; force) do rootfs + rootfs_url = "https://github.com/alpinelinux/docker-alpine/raw/v$(release.major).$(release.minor)/x86_64/alpine-$(variant)-$(release)-x86_64.tar.gz" + @info("Downloading Alpine rootfs", url=rootfs_url) + rm(rootfs) + Pkg.Artifacts.download_verify_unpack(rootfs_url, nothing, rootfs; verbose=true) + + # Call user callback, if requested + f(rootfs) + + # Remove special `dev` files, take ownership, force symlinks to be relative, etc... + rootfs_info = """ + rootfs_type=alpine + release=$(release) + variant=$(variant) + packages=$(join([pkg.name for pkg in packages], ",")) + build_date=$(Dates.now()) + """ + cleanup_rootfs(rootfs; rootfs_info) + + # Generate one `apk` invocation per repository + repos = unique([pkg.repo for pkg in packages]) + for repo in repos + apk_args = ["/sbin/apk", "add", "--no-chown"] + if repo !== nothing + push!(apk_args, repository_arg(repo)) + end + for pkg in filter(pkg -> pkg.repo == repo, packages) + push!(apk_args, pkg.name) + end + chroot(rootfs, apk_args...) + end + end +end +# If no user callback is provided, default to doing nothing +alpine_bootstrap(name::String; kwargs...) = alpine_bootstrap(p -> nothing, name; kwargs...) diff --git a/src/build/args.jl b/src/build/args.jl new file mode 100644 index 0000000..9018f51 --- /dev/null +++ b/src/build/args.jl @@ -0,0 +1,10 @@ +function parse_build_args(args::AbstractVector) + settings = ArgParse.ArgParseSettings() + ArgParse.@add_arg_table! settings begin + "--arch" + required=true + end + parsed_args = ArgParse.parse_args(args, settings) + arch = parsed_args["arch"]::String + return (; arch) +end diff --git a/rootfs_utils.jl b/src/build/common.jl similarity index 54% rename from rootfs_utils.jl rename to src/build/common.jl index 14af826..8f369da 100644 --- a/rootfs_utils.jl +++ b/src/build/common.jl @@ -1,29 +1,3 @@ -using ArgParse: ArgParse -using Base.BinaryPlatforms -using Dates -using Pkg -using Pkg.Artifacts -using SHA -using Scratch -using Test -using ghr_jll - -# Utility functions -getuid() = ccall(:getuid, Cint, ()) -getgid() = ccall(:getgid, Cint, ()) -chroot(rootfs, cmds...; uid=getuid(), gid=getgid()) = run(`sudo chroot --userspec=$(uid):$(gid) $(rootfs) $(cmds)`) - -function parse_args(args::AbstractVector) - settings = ArgParse.ArgParseSettings() - ArgParse.@add_arg_table! settings begin - "--arch" - required=true - end - parsed_args = ArgParse.parse_args(args, settings) - arch = parsed_args["arch"]::String - return (; arch) -end - # Sometimes rootfs images have absolute symlinks within them; this # is very bad for us as it breaks our ability to look at a rootfs # without `chroot`'ing into it; so we fix up all the links to be @@ -37,6 +11,7 @@ function force_relative(link, rootfs) rm(link; force=true) symlink(relpath(target, dirname(link)), link) end + function force_relative(rootfs) for (root, dirs, files) in walkdir(rootfs) for f in files @@ -144,17 +119,6 @@ function normalize_arch(image_arch::String) end end -function debian_arch(image_arch::String) - debian_arch_mapping = Dict( - "x86_64" => "amd64", - "i686" => "i386", - "armv7l" => "armhf", - "aarch64" => "arm64", - "powerpc64le" => "ppc64el", - ) - return debian_arch_mapping[normalize_arch(image_arch)] -end - function can_run_natively(image_arch::String) native_arch = Base.BinaryPlatforms.arch(HostPlatform()) if native_arch == "x86_64" @@ -179,119 +143,6 @@ function qemu_installed(image_arch::String) return Sys.which("qemu-$(qemu_arch_mapping[image_arch])-static") !== nothing end -function debootstrap(f::Function, arch::String, name::String; - release::String="buster", - variant::String="minbase", - packages::Vector{String}=String[], - force::Bool=false) - if Sys.which("debootstrap") === nothing - error("Must install `debootstrap`!") - end - - arch = normalize_arch(arch) - if !can_run_natively(arch) && !qemu_installed(arch) - error("Must install qemu-user-static and binfmt_misc!") - end - - return create_rootfs(name; force) do rootfs - packages_string = join(push!(packages, "locales"), ",") - @info("Running debootstrap", release, variant, packages) - run(`sudo debootstrap --arch=$(debian_arch(arch)) --variant=$(variant) --include=$(packages_string) $(release) "$(rootfs)"`) - - # This is necessary on any 32-bit userspaces to work around the - # following bad interaction between qemu, linux and openssl: - # https://serverfault.com/questions/1045118/debootstrap-armhd-buster-unable-to-get-local-issuer-certificate - if isfile(joinpath(rootfs, "usr", "bin", "c_rehash")) - chroot(rootfs, "/usr/bin/c_rehash"; uid=0, gid=0) - end - - # Call user callback, if requested - f(rootfs) - - # Remove special `dev` files, take ownership, force symlinks to be relative, etc... - rootfs_info=""" - rootfs_type=debootstrap - release=$(release) - variant=$(variant) - packages=$(packages_string) - build_date=$(Dates.now()) - """ - cleanup_rootfs(rootfs; rootfs_info) - - # Remove `_apt` user so that `apt` doesn't try to `setgroups()` - @info("Removing `_apt` user") - open(joinpath(rootfs, "etc", "passwd"), write=true, read=true) do io - filtered_lines = filter(l -> !startswith(l, "_apt:"), readlines(io)) - truncate(io, 0) - seek(io, 0) - for l in filtered_lines - println(io, l) - end - end - - # Set up the one true locale - @info("Setting up UTF-8 locale") - open(joinpath(rootfs, "etc", "locale.gen"), "a") do io - println(io, "en_US.UTF-8 UTF-8") - end - chroot(rootfs, "locale-gen") - end -end -# If no user callback is provided, default to doing nothing -debootstrap(arch::String, name::String; kwargs...) = debootstrap(p -> nothing, arch, name; kwargs...) - -# Helper structure for installing alpine packages that may or may not be part of an older Alpine release -struct AlpinePackage - name::String - repo::Union{Nothing,String} - - AlpinePackage(name, repo=nothing) = new(name, repo) -end -function repository_arg(repo) - if startswith(repo, "https://") - return "--repository=$(repo)" - end - return "--repository=http://dl-cdn.alpinelinux.org/alpine/$(repo)/main" -end - -function alpine_bootstrap(f::Function, name::String; release::VersionNumber=v"3.13.5", variant="minirootfs", - packages::Vector{AlpinePackage}=AlpinePackage[], force::Bool=false) - return create_rootfs(name; force) do rootfs - rootfs_url = "https://github.com/alpinelinux/docker-alpine/raw/v$(release.major).$(release.minor)/x86_64/alpine-$(variant)-$(release)-x86_64.tar.gz" - @info("Downloading Alpine rootfs", url=rootfs_url) - rm(rootfs) - Pkg.Artifacts.download_verify_unpack(rootfs_url, nothing, rootfs; verbose=true) - - # Call user callback, if requested - f(rootfs) - - # Remove special `dev` files, take ownership, force symlinks to be relative, etc... - rootfs_info = """ - rootfs_type=alpine - release=$(release) - variant=$(variant) - packages=$(join([pkg.name for pkg in packages], ",")) - build_date=$(Dates.now()) - """ - cleanup_rootfs(rootfs; rootfs_info) - - # Generate one `apk` invocation per repository - repos = unique([pkg.repo for pkg in packages]) - for repo in repos - apk_args = ["/sbin/apk", "add", "--no-chown"] - if repo !== nothing - push!(apk_args, repository_arg(repo)) - end - for pkg in filter(pkg -> pkg.repo == repo, packages) - push!(apk_args, pkg.name) - end - chroot(rootfs, apk_args...) - end - end -end -# If no user callback is provided, default to doing nothing -alpine_bootstrap(name::String; kwargs...) = alpine_bootstrap(p -> nothing, name; kwargs...) - function upload_rootfs_image(tarball_path::String; github_repo::String, tag_name::String, @@ -374,24 +225,3 @@ function upload_rootfs_image_github_actions(tarball_path::String) tag_name, ) end - -function test_sandbox(artifact_hash) - test_cmd = `$(Base.julia_cmd())` - push!(test_cmd.exec, "--project=$(Base.active_project())") - push!(test_cmd.exec, joinpath(@__DIR__, "test_rootfs.jl")) - push!(test_cmd.exec, "") - push!(test_cmd.exec, "$(artifact_hash)") - push!(test_cmd.exec, "/bin/bash") - push!(test_cmd.exec, "-c") - push!(test_cmd.exec, "echo Hello from inside the sandbox") - @testset "Test sandbox" begin - @testset begin - run(test_cmd) - end - @testset begin - @test success(test_cmd) - @test read(test_cmd, String) == "Hello from inside the sandbox\n" - end - end - return nothing -end diff --git a/src/build/debian.jl b/src/build/debian.jl new file mode 100644 index 0000000..099acdb --- /dev/null +++ b/src/build/debian.jl @@ -0,0 +1,72 @@ +function debian_arch(image_arch::String) + debian_arch_mapping = Dict( + "x86_64" => "amd64", + "i686" => "i386", + "armv7l" => "armhf", + "aarch64" => "arm64", + "powerpc64le" => "ppc64el", + ) + return debian_arch_mapping[normalize_arch(image_arch)] +end + +function debootstrap(f::Function, arch::String, name::String; + release::String="buster", + variant::String="minbase", + packages::Vector{String}=String[], + force::Bool=false) + if Sys.which("debootstrap") === nothing + error("Must install `debootstrap`!") + end + + arch = normalize_arch(arch) + if !can_run_natively(arch) && !qemu_installed(arch) + error("Must install qemu-user-static and binfmt_misc!") + end + + return create_rootfs(name; force) do rootfs + packages_string = join(push!(packages, "locales"), ",") + @info("Running debootstrap", release, variant, packages) + run(`sudo debootstrap --arch=$(debian_arch(arch)) --variant=$(variant) --include=$(packages_string) $(release) "$(rootfs)"`) + + # This is necessary on any 32-bit userspaces to work around the + # following bad interaction between qemu, linux and openssl: + # https://serverfault.com/questions/1045118/debootstrap-armhd-buster-unable-to-get-local-issuer-certificate + if isfile(joinpath(rootfs, "usr", "bin", "c_rehash")) + chroot(rootfs, "/usr/bin/c_rehash"; uid=0, gid=0) + end + + # Call user callback, if requested + f(rootfs) + + # Remove special `dev` files, take ownership, force symlinks to be relative, etc... + rootfs_info=""" + rootfs_type=debootstrap + release=$(release) + variant=$(variant) + packages=$(packages_string) + build_date=$(Dates.now()) + """ + cleanup_rootfs(rootfs; rootfs_info) + + # Remove `_apt` user so that `apt` doesn't try to `setgroups()` + @info("Removing `_apt` user") + open(joinpath(rootfs, "etc", "passwd"), write=true, read=true) do io + filtered_lines = filter(l -> !startswith(l, "_apt:"), readlines(io)) + truncate(io, 0) + seek(io, 0) + for l in filtered_lines + println(io, l) + end + end + + # Set up the one true locale + @info("Setting up UTF-8 locale") + open(joinpath(rootfs, "etc", "locale.gen"), "a") do io + println(io, "en_US.UTF-8 UTF-8") + end + chroot(rootfs, "locale-gen") + end +end + +# If no user callback is provided, default to doing nothing +debootstrap(arch::String, name::String; kwargs...) = debootstrap(p -> nothing, arch, name; kwargs...) diff --git a/src/build/utils.jl b/src/build/utils.jl new file mode 100644 index 0000000..5333b1b --- /dev/null +++ b/src/build/utils.jl @@ -0,0 +1,4 @@ +# Utility functions +getuid() = ccall(:getuid, Cint, ()) +getgid() = ccall(:getgid, Cint, ()) +chroot(rootfs, cmds...; uid=getuid(), gid=getgid()) = run(`sudo chroot --userspec=$(uid):$(gid) $(rootfs) $(cmds)`) diff --git a/src/run/args.jl b/src/run/args.jl new file mode 100644 index 0000000..e69de29 diff --git a/src/run/test.jl b/src/run/test.jl new file mode 100644 index 0000000..9833fc0 --- /dev/null +++ b/src/run/test.jl @@ -0,0 +1,20 @@ +function test_sandbox(artifact_hash) + test_cmd = `$(Base.julia_cmd())` + push!(test_cmd.exec, "--project=$(Base.active_project())") + push!(test_cmd.exec, joinpath(dirname(dirname(@__DIR__)), "test_rootfs.jl")) + push!(test_cmd.exec, "") + push!(test_cmd.exec, "$(artifact_hash)") + push!(test_cmd.exec, "/bin/bash") + push!(test_cmd.exec, "-c") + push!(test_cmd.exec, "echo Hello from inside the sandbox") + @testset "Test sandbox" begin + @testset begin + run(test_cmd) + end + @testset begin + @test success(test_cmd) + @test read(test_cmd, String) == "Hello from inside the sandbox\n" + end + end + return nothing +end diff --git a/src/types.jl b/src/types.jl new file mode 100644 index 0000000..8b59b01 --- /dev/null +++ b/src/types.jl @@ -0,0 +1,7 @@ +# Helper structure for installing alpine packages that may or may not be part of an older Alpine release +struct AlpinePackage + name::String + repo::Union{Nothing,String} + + AlpinePackage(name, repo=nothing) = new(name, repo) +end