Reorganize the repository code to make it a little bit more package-l…

…ike (#66)

.github/workflows/build.yml

@@ -83,8 +83,9 @@ jobs:
       - run: sudo apt-get update
       - run: sudo apt-get -y install binfmt-support debootstrap qemu-user-static
       - run: update-binfmts --display
-      - run: julia --color=yes --project=. -e 'using Pkg; Pkg.instantiate()'
-      - run: julia --color=yes --project=. -e 'using Pkg; Pkg.precompile()'
+      - run: julia --color=yes --project=. -e 'using Pkg; @time Pkg.instantiate()'
+      - run: julia --color=yes --project=. -e 'using Pkg; @time Pkg.precompile()'
+      - run: julia --color=yes --project=. -e '@time using RootfsUtils'
       - run: |
           IMAGE_NAME=$(echo ${{ matrix.image }} | cut -d. -f1)
           IMAGE_ARCH=$(echo ${{ matrix.image }} | cut -d. -f2)

Manifest.toml

@@ -124,6 +124,10 @@ uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 deps = ["ArgTools", "SHA"]
 uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 
+[[Test]]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
 [[TextWrap]]
 git-tree-sha1 = "9250ef9b01b66667380cf3275b3f7488d0e25faf"
 uuid = "b718987f-49a8-5099-9789-dcd902bef87d"

Project.toml

@@ -1,9 +1,14 @@
+name = "RootfsUtils"
+uuid = "ea0daf2f-8d67-45cc-b2d9-fd898f843992"
+
 [deps]
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 Sandbox = "9307e30f-c43e-9ca7-d17c-c2dc59df670d"
 Scratch = "6c6a2e73-6563-6170-7368-637461726353"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
 
 [compat]

images/agent_linux.jl

@@ -2,8 +2,9 @@
 ## to run inside of.  Most CI steps will be run within a different image
 ## nested inside of this one.
 
-include(joinpath(dirname(@__DIR__), "rootfs_utils.jl"))
-arch, = parse_args(ARGS)
+using RootfsUtils
+
+arch, = parse_build_args(ARGS)
 image = "$(splitext(basename(@__FILE__))[1]).$(arch)"
 
 # Build debian-based image with the following extra packages:

images/llvm_passes.jl

@@ -1,7 +1,8 @@
 ## This rootfs includes enough of a host toolchain to build the LLVM passes (such as `analyzegc`).
 
-include(joinpath(dirname(@__DIR__), "rootfs_utils.jl"))
-arch, = parse_args(ARGS)
+using RootfsUtils
+
+arch, = parse_build_args(ARGS)
 image = "$(splitext(basename(@__FILE__))[1]).$(arch)"
 
 # Build debian-based image with the following extra packages:

images/package_linux.jl

@@ -1,8 +1,9 @@
 ## This rootfs includes everything that must be installed to build Julia
 ## within a debian-based environment with GCC 9.
 
-include(joinpath(dirname(@__DIR__), "rootfs_utils.jl"))
-arch, = parse_args(ARGS)
+using RootfsUtils
+
+arch, = parse_build_args(ARGS)
 image = "$(splitext(basename(@__FILE__))[1]).$(arch)"
 
 # Build debian-based image with the following extra packages:

images/package_linux_qemu.jl

@@ -1,8 +1,9 @@
 ## This rootfs includes everything that must be installed to build Julia
 ## within a debian-based environment with GCC 9.
 
-include(joinpath(dirname(@__DIR__), "rootfs_utils.jl"))
-arch, = parse_args(ARGS)
+using RootfsUtils
+
+arch, = parse_build_args(ARGS)
 image = "$(splitext(basename(@__FILE__))[1]).$(arch)"
 
 # Build debian-based image with the following extra packages:

images/package_musl.jl

@@ -1,8 +1,9 @@
 ## This rootfs includes everything that must be installed to build Julia
 ## within an alpine-based environment with GCC 9.
 
-include(joinpath(dirname(@__DIR__), "rootfs_utils.jl"))
-arch, = parse_args(ARGS)
+using RootfsUtils
+
+arch, = parse_build_args(ARGS)
 image = "$(splitext(basename(@__FILE__))[1]).$(arch)"
 
 # Build alpine-based image with the following extra packages:

images/tester_linux.jl

@@ -1,7 +1,8 @@
 ## This rootfs does not include the compiler toolchain.
 
-include(joinpath(dirname(@__DIR__), "rootfs_utils.jl"))
-arch, = parse_args(ARGS)
+using RootfsUtils
+
+arch, = parse_build_args(ARGS)
 image = "$(splitext(basename(@__FILE__))[1]).$(arch)"
 
 # Build debian-based image with the following extra packages:

images/tester_musl.jl

@@ -1,7 +1,8 @@
 ## This rootfs does not include the compiler toolchain.
 
-include(joinpath(dirname(@__DIR__), "rootfs_utils.jl"))
-arch, = parse_args(ARGS)
+using RootfsUtils
+
+arch, = parse_build_args(ARGS)
 image = "$(splitext(basename(@__FILE__))[1]).$(arch)"
 
 # Build alpine-based image with the following extra packages:

src/RootfsUtils.jl

@@ -0,0 +1,32 @@
+module RootfsUtils
+
+using ArgParse: ArgParse
+using Base.BinaryPlatforms
+using Dates
+using Pkg
+using Pkg.Artifacts
+using SHA
+using Scratch
+using Test
+using ghr_jll
+
+export AlpinePackage
+export alpine_bootstrap
+export chroot
+export debootstrap
+export parse_build_args
+export test_sandbox
+export upload_rootfs_image_github_actions
+
+include("types.jl")
+
+include("build/alpine.jl")
+include("build/args.jl")
+include("build/common.jl")
+include("build/debian.jl")
+include("build/utils.jl")
+
+include("run/args.jl")
+include("run/test.jl")
+
+end # module

src/build/alpine.jl

@@ -0,0 +1,44 @@
+function repository_arg(repo)
+    if startswith(repo, "https://")
+        return "--repository=$(repo)"
+    end
+    return "--repository=http://dl-cdn.alpinelinux.org/alpine/$(repo)/main"
+end
+
+function alpine_bootstrap(f::Function, name::String; release::VersionNumber=v"3.13.5", variant="minirootfs",
+                          packages::Vector{AlpinePackage}=AlpinePackage[], force::Bool=false)
+    return create_rootfs(name; force) do rootfs
+        rootfs_url = "https://github.com/alpinelinux/docker-alpine/raw/v$(release.major).$(release.minor)/x86_64/alpine-$(variant)-$(release)-x86_64.tar.gz"
+        @info("Downloading Alpine rootfs", url=rootfs_url)
+        rm(rootfs)
+        Pkg.Artifacts.download_verify_unpack(rootfs_url, nothing, rootfs; verbose=true)
+
+        # Call user callback, if requested
+        f(rootfs)
+
+        # Remove special `dev` files, take ownership, force symlinks to be relative, etc...
+        rootfs_info = """
+                    rootfs_type=alpine
+                    release=$(release)
+                    variant=$(variant)
+                    packages=$(join([pkg.name for pkg in packages], ","))
+                    build_date=$(Dates.now())
+                    """
+        cleanup_rootfs(rootfs; rootfs_info)
+
+        # Generate one `apk` invocation per repository
+        repos = unique([pkg.repo for pkg in packages])
+        for repo in repos
+            apk_args = ["/sbin/apk", "add", "--no-chown"]
+            if repo !== nothing
+                push!(apk_args, repository_arg(repo))
+            end
+            for pkg in filter(pkg -> pkg.repo == repo, packages)
+                push!(apk_args, pkg.name)
+            end
+            chroot(rootfs, apk_args...)
+        end
+    end
+end
+# If no user callback is provided, default to doing nothing
+alpine_bootstrap(name::String; kwargs...) = alpine_bootstrap(p -> nothing, name; kwargs...)

src/build/args.jl

@@ -0,0 +1,10 @@
+function parse_build_args(args::AbstractVector)
+    settings = ArgParse.ArgParseSettings()
+    ArgParse.@add_arg_table! settings begin
+        "--arch"
+            required=true
+    end
+    parsed_args = ArgParse.parse_args(args, settings)
+    arch = parsed_args["arch"]::String
+    return (; arch)
+end