From af1973a4d644b84c518f88dc804ef847e3761263 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 3 Sep 2020 14:14:10 -0400 Subject: [PATCH 1/3] Add support for parallel universes This adds the concept of a "universe" for autobuilds. A universe is essentially a depot that has its own copy of the General registry. When building packages using `--deploy=universe` the built jll gets registered into this copy of the registry and subsequent builds can pick it up from there as if it had been registered with the real general registry. This allows successive builds of dependent packages without first having to go register the dependency. The universe to use is specified using `--universe=`. If a universe by that name does not exist, it is created. Otherwise the previously created universe is re-used. --- Manifest.toml | 8 +- Project.toml | 1 + src/AutoBuild.jl | 261 +++++++++++++++++++++++++++++++++-------------- 3 files changed, 192 insertions(+), 78 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 3c58be2c5..a3ee5f373 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -22,7 +22,7 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" [[BinaryBuilderBase]] deps = ["CodecZlib", "JSON", "LibGit2", "Libdl", "Logging", "OutputCollectors", "Pkg", "Random", "SHA", "UUIDs"] -git-tree-sha1 = "97cd717e1e3453db6211214b3041f120dc8d0816" +git-tree-sha1 = "9f94fe7279ee3565d793f2a157ac369fb1da62e2" repo-rev = "master" repo-url = "https://github.com/JuliaPackaging/BinaryBuilderBase.jl.git" uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e" @@ -141,6 +141,12 @@ uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" deps = ["Libdl"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +[[LocalRegistry]] +deps = ["Pkg", "Random", "RegistryTools", "UUIDs"] +git-tree-sha1 = "995d6f723eb3b89b62ae65cc0ca8a8063ebfa8ad" +uuid = "89398ba2-070a-4b16-a995-9893c55d93cf" +version = "0.3.0" + [[Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" diff --git a/Project.toml b/Project.toml index 0c4f9731e..5d3c2bd65 100644 --- a/Project.toml +++ b/Project.toml @@ -14,6 +14,7 @@ JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +LocalRegistry = "89398ba2-070a-4b16-a995-9893c55d93cf" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36" ObjectFile = "d8793406-e978-5875-9003-1fc021f44a92" diff --git a/src/AutoBuild.jl b/src/AutoBuild.jl index 834489352..3f3cb4345 100644 --- a/src/AutoBuild.jl +++ b/src/AutoBuild.jl @@ -2,7 +2,7 @@ export build_tarballs, autobuild, print_artifacts_toml, build, get_meta_json import GitHub: gh_get_json, DEFAULT_API import SHA: sha256, sha1 using Pkg.TOML, Dates, UUIDs -using RegistryTools, Registrator +using RegistryTools, LocalRegistry import LibGit2 import PkgLicenses @@ -13,7 +13,7 @@ const BUILD_HELP = ( Usage: build_tarballs.jl [target1,target2,...] [--help] [--verbose] [--debug] [--deploy] [--deploy-bin] [--deploy-jll] - [--register] [--meta-json] + [--register] [--meta-json] [--universe=] Options: targets By default `build_tarballs.jl` will build a tarball @@ -57,6 +57,14 @@ const BUILD_HELP = ( (and often does) output multiple JSON objects for multiple platforms, multi-stage builds, etc... + --universe= A "universe" is an implicit depot, keyed on a name. + Universes are implicitly created upon first use of + the name. Subsequent uses of the same name will re-use + the universe. Use `--deploy=universe` to deploy the + package into the selected universe (otherwise the + universe is used only for dependencies). This option + is useful to build nested dependency trees locally. + --help Print out this message. Examples: @@ -134,6 +142,10 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, deploy, deploy_repo = extract_flag!(ARGS, "--deploy", "JuliaBinaryWrappers/$(src_name)_jll.jl") deploy_bin, deploy_bin_repo = extract_flag!(ARGS, "--deploy-bin", "JuliaBinaryWrappers/$(src_name)_jll.jl") deploy_jll, deploy_jll_repo = extract_flag!(ARGS, "--deploy-jll", "JuliaBinaryWrappers/$(src_name)_jll.jl") + deploy_universe = false + + # This sets which universe to use for dependencies + universe, universe_name = extract_flag!(ARGS, "--universe", nothing) # Resolve deploy settings if deploy @@ -160,8 +172,57 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, error("Cannot register with a local deployment!") end + universe_path = nothing + if universe + if deploy + if deploy_repo != "universe" + error("When deploying from a universe, must use --deploy=universe") + end + deploy_universe = true + deploy_bin_repo = deploy_jll_repo = "JuliaBinaryWrappers/$(src_name)_jll.jl" + end + + # Find the universe directory by ascending from pwd() until we find a "universes" directory + dir = pwd() + while !isdir(joinpath(dir, "universes")) + dir = dirname(dir) + if dir == "/" + error("Could not find `universes` diretory in any parent of pwd") + end + end + universe_path = joinpath(dir, "universes", universe_name) + + if !isdir(universe_path) + if deploy_universe + mkpath(universe_path) + + # Symlink the `packages`, `logs` and `artifacts` directories to the + # currently active repo + active_depot = first(Base.DEPOT_PATH) + for folder in ("packages", "logs", "artifacts") + symlink(joinpath(active_depot, folder), joinpath(universe_path, folder)) + end + + # Create new `registries` and `clones` folders local to this registry + mkpath(joinpath(universe_path, "registries")) + mkpath(joinpath(universe_path, "clones")) + + ctx = Pkg.Types.Context() + Pkg.GitTools.clone(ctx, "https://github.com/JuliaRegistries/General", + joinpath(universe_path, "registries", "General"); header = "General registry to new universe") + else + @warn "Universe does not exist, but deploy not specified. Skipping Universe creation." + universe_path = nothing + end + end + end + if deploy_bin || deploy_jll - code_dir = joinpath(Pkg.devdir(), "$(src_name)_jll") + if universe + code_dir = joinpath(universe_path, "clones", string(jll_uuid("$(src_name)_jll"))) + else + code_dir = joinpath(Pkg.devdir(), "$(src_name)_jll") + end # Shove them into `kwargs` so that we are conditionally passing them along kwargs = (; kwargs..., code_dir = code_dir) @@ -196,10 +257,10 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, end # Check to make sure we have the necessary environment stuff - if deploy_bin || deploy_jll + if deploy_bin || deploy_jll || deploy_universe # Check to see if we've already got a wrapper package within the Registry, # choose a version number that is greater than anything else existent. - build_version = get_next_wrapper_version(src_name, src_version) + build_version = get_next_wrapper_version(src_name, src_version; universe = universe ? universe_path : nothing) if verbose @info("Building and deploying version $(build_version) to $(deploy_jll_repo)") end @@ -208,7 +269,7 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, # We need to make sure that the JLL repo at least exists, so that we can deploy binaries to it # even if we're not planning to register things to it today. if deploy_jll_repo != "local" - init_jll_package(src_name, code_dir, deploy_jll_repo) + init_jll_package(src_name, code_dir, deploy_jll_repo; gh_create = !deploy_universe) end end @@ -243,21 +304,23 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, build_output_meta = Dict() else - # Build the given platforms using the given sources - build_output_meta = autobuild( - # Controls output product placement, mount directory placement, etc... - pwd(), - - args...; - - # Flags - verbose=verbose, - debug=debug, - kwargs..., - ) + in_universe(universe_path) do ctx + # Build the given platforms using the given sources + build_output_meta = autobuild( + # Controls output product placement, mount directory placement, etc... + pwd(), + + args...; + + # Flags + verbose=verbose, + debug=debug, + kwargs..., + ) + end end - if deploy_jll + if deploy_jll || deploy_universe if verbose @info("Committing and pushing $(src_name)_jll.jl wrapper code version $(build_version)...") end @@ -270,7 +333,7 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, bin_path = "https://github.com/$(deploy_jll_repo)/releases/download/$(tag)" build_jll_package(src_name, build_version, sources, code_dir, build_output_meta, dependencies, bin_path; verbose=verbose, extra_kwargs...) - if deploy_jll_repo != "local" + if deploy_jll_repo != "local" && !deploy_universe push_jll_package(src_name, build_version; code_dir=code_dir, deploy_repo=deploy_jll_repo) end if register @@ -281,9 +344,14 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, register_jll(src_name, build_version, dependencies, julia_compat; deploy_repo=deploy_jll_repo, code_dir=code_dir) end + if deploy_universe + @info("Registering package in local universe") + commit_jll_package(src_name, code_dir, build_version) + LocalRegistry.register(code_dir, joinpath(universe_path, "registries", "General")) + end end - if deploy_bin && deploy_bin_repo != "local" + if deploy_bin && deploy_bin_repo != "local" && !deploy_universe # Upload the binaries if verbose @info("Deploying binaries to release $(tag) on $(deploy_bin_repo) via `ghr`...") @@ -380,37 +448,57 @@ if VERSION < v"1.4-" Pkg.Operations.registered_paths(ctx::Pkg.Types.Context, uuid::UUID) = Pkg.Operations.registered_paths(ctx.env, uuid) end -function get_next_wrapper_version(src_name, src_version) +function in_universe(f, universe; update = false) + if universe === nothing + ctx = Pkg.Types.Context() + # Force-update the registry here, since we may have pushed a new version recently + update && update_registry(ctx, devnull) + else + old_depot_path = copy(Base.DEPOT_PATH) + empty!(Base.DEPOT_PATH) + push!(Base.DEPOT_PATH, universe) + ctx = Pkg.Types.Context() + end + + try + f(ctx) + finally + if universe !== nothing + empty!(Base.DEPOT_PATH) + append!(Base.DEPOT_PATH, old_depot_path) + end + end +end + +function get_next_wrapper_version(src_name, src_version; universe=nothing) # If src_version already has a build_number, just return it immediately if src_version.build != () return src_version end - ctx = Pkg.Types.Context() - - # Force-update the registry here, since we may have pushed a new version recently - update_registry(ctx, devnull) - - # If it does, we need to bump the build number up to the next value - build_number = 0 - if any(isfile(joinpath(p, "Package.toml")) for p in Pkg.Operations.registered_paths(ctx, jll_uuid("$(src_name)_jll"))) - # Find largest version number that matches ours in the registered paths - versions = VersionNumber[] - for path in Pkg.Operations.registered_paths(ctx, jll_uuid("$(src_name)_jll")) - append!(versions, RegistryTools.Compress.load_versions(joinpath(path, "Versions.toml"))) - end - versions = filter(v -> (v.major == src_version.major) && - (v.minor == src_version.minor) && - (v.patch == src_version.patch) && - (v.build isa Tuple{<:UInt}), versions) - # Our build number must be larger than the maximum already present in the registry - if !isempty(versions) - build_number = first(maximum(versions).build) + 1 + + in_universe(universe) do ctx + # If it does, we need to bump the build number up to the next value + build_number = 0 + if any(isfile(joinpath(p, "Package.toml")) for p in Pkg.Operations.registered_paths(ctx, jll_uuid("$(src_name)_jll"))) + # Find largest version number that matches ours in the registered paths + versions = VersionNumber[] + for path in Pkg.Operations.registered_paths(ctx, jll_uuid("$(src_name)_jll")) + append!(versions, RegistryTools.Compress.load_versions(joinpath(path, "Versions.toml"))) + end + versions = filter(v -> (v.major == src_version.major) && + (v.minor == src_version.minor) && + (v.patch == src_version.patch) && + (v.build isa Tuple{<:UInt}), versions) + # Our build number must be larger than the maximum already present in the registry + if !isempty(versions) + build_number = first(maximum(versions).build) + 1 + end end - end - # Construct build_version (src_version + build_number) - build_version = VersionNumber(src_version.major, src_version.minor, - src_version.patch, src_version.prerelease, (build_number,)) + # Construct build_version (src_version + build_number) + build_version = VersionNumber(src_version.major, src_version.minor, + src_version.patch, src_version.prerelease, (build_number,)) + end end function _registered_packages(registry_url::AbstractString) @@ -856,47 +944,61 @@ end function init_jll_package(name, code_dir, deploy_repo; - gh_auth = Wizard.github_auth(;allow_anonymous=false), - gh_username = gh_get_json(DEFAULT_API, "/user"; auth=gh_auth)["login"]) + gh_create = false, + gh_auth = gh_create ? Wizard.github_auth(;allow_anonymous=false) : nothing, + gh_username = gh_create ? gh_get_json(DEFAULT_API, "/user"; auth=gh_auth)["login"] : nothing) + gh_repo = nothing try # This throws if it does not exist - GitHub.repo(deploy_repo; auth=gh_auth) + gh_repo = GitHub.repo(deploy_repo; auth=gh_auth) catch e - # If it doesn't exist, create it. - # check whether gh_org might be a user, not an organization. - gh_org = dirname(deploy_repo) - isorg = GitHub.owner(gh_org; auth=gh_auth).typ == "Organization" - owner = GitHub.Owner(gh_org, isorg) - @info("Creating new wrapper code repo at https://github.com/$(deploy_repo)") - try - GitHub.create_repo(owner, basename(deploy_repo), Dict("license_template" => "mit", "has_issues" => "false"); auth=gh_auth) - catch create_e - # If creation failed, it could be because the repo was created in the meantime. - # Check for that; if it still doesn't exist, then freak out. Otherwise, continue on. + if gh_create + # If it doesn't exist, create it. + # check whether gh_org might be a user, not an organization. + gh_org = dirname(deploy_repo) + isorg = GitHub.owner(gh_org; auth=gh_auth).typ == "Organization" + owner = GitHub.Owner(gh_org, isorg) + @info("Creating new wrapper code repo at https://github.com/$(deploy_repo)") try - GitHub.repo(deploy_repo; auth=gh_auth) - catch - rethrow(create_e) + GitHub.create_repo(owner, basename(deploy_repo), Dict("license_template" => "mit", "has_issues" => "false"); auth=gh_auth) + catch create_e + # If creation failed, it could be because the repo was created in the meantime. + # Check for that; if it still doesn't exist, then freak out. Otherwise, continue on. + try + gh_repo = GitHub.repo(deploy_repo; auth=gh_auth) + catch + rethrow(create_e) + end end end end if !isdir(code_dir) - # If it does exist, clone it down: - @info("Cloning wrapper code repo from https://github.com/$(deploy_repo) into $(code_dir)") - Wizard.with_gitcreds(gh_username, gh_auth.token) do creds - LibGit2.clone("https://github.com/$(deploy_repo)", code_dir; credentials=creds) + if gh_repo === nothing + # The gh repo didn't exist and we didn't want to create it. Initialize an empty repository + mkpath(code_dir) + repo = LibGit2.init(code_dir) + LibGit2.commit(repo, "Initial empty commit") + else + # If it does exist, clone it down: + @info("Cloning wrapper code repo from https://github.com/$(deploy_repo) into $(code_dir)") + Wizard.with_gitcreds(gh_auth) do creds + LibGit2.clone("https://github.com/$(deploy_repo)", code_dir; credentials=creds) + end end else # Otherwise, hard-reset to latest master: repo = LibGit2.GitRepo(code_dir) - Wizard.with_gitcreds(gh_username, gh_auth.token) do creds - LibGit2.fetch(repo; credentials=creds) - end - origin_master_oid = LibGit2.GitHash(LibGit2.lookup_branch(repo, "origin/master", true)) - LibGit2.reset!(repo, origin_master_oid, LibGit2.Consts.RESET_HARD) - if string(LibGit2.head_oid(repo)) != string(origin_master_oid) - LibGit2.branch!(repo, "master", string(origin_master_oid); force=true) + # ... but only if this repo actually has an upstream + if LibGit2.lookup_branch(repo, "origin/master", true) !== nothing + Wizard.with_gitcreds(gh_auth) do creds + LibGit2.fetch(repo; credentials=creds) + end + origin_master_oid = LibGit2.GitHash(LibGit2.lookup_branch(repo, "origin/master", true)) + LibGit2.reset!(repo, origin_master_oid, LibGit2.Consts.RESET_HARD) + if string(LibGit2.head_oid(repo)) != string(origin_master_oid) + LibGit2.branch!(repo, "master", string(origin_master_oid); force=true) + end end end end @@ -1466,15 +1568,20 @@ function build_jll_package(src_name::String, end end +function commit_jll_package(name, code_dir, build_version) + wrapper_repo = LibGit2.GitRepo(code_dir) + LibGit2.add!(wrapper_repo, ".") + LibGit2.commit(wrapper_repo, "$(name)_jll build $(build_version)") + return wrapper_repo +end + function push_jll_package(name, build_version; code_dir = joinpath(Pkg.devdir(), "$(name)_jll"), deploy_repo = "JuliaBinaryWrappers/$(name)_jll.jl", gh_auth = Wizard.github_auth(;allow_anonymous=false), gh_username = gh_get_json(DEFAULT_API, "/user"; auth=gh_auth)["login"]) + wrapper_repo = commit_jll_package(name, code_dir, build_version) # Next, push up the wrapper code repository - wrapper_repo = LibGit2.GitRepo(code_dir) - LibGit2.add!(wrapper_repo, ".") - LibGit2.commit(wrapper_repo, "$(name)_jll build $(build_version)") Wizard.with_gitcreds(gh_username, gh_auth.token) do creds LibGit2.push( wrapper_repo; From 352d2e88e6ea1b8cc4edc76172bee0c0c02ec3da Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 3 Sep 2020 20:10:34 -0400 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Fredrik Ekre --- src/AutoBuild.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/AutoBuild.jl b/src/AutoBuild.jl index 3f3cb4345..13f407d00 100644 --- a/src/AutoBuild.jl +++ b/src/AutoBuild.jl @@ -197,10 +197,10 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, mkpath(universe_path) # Symlink the `packages`, `logs` and `artifacts` directories to the - # currently active repo - active_depot = first(Base.DEPOT_PATH) + # user depot to take advantage of packages already installed there + user_depot = first(Base.DEPOT_PATH) for folder in ("packages", "logs", "artifacts") - symlink(joinpath(active_depot, folder), joinpath(universe_path, folder)) + symlink(joinpath(user_depot, folder), joinpath(universe_path, folder)) end # Create new `registries` and `clones` folders local to this registry From b553367d348055b3a9bc96b7a476ad5ad997f237 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 3 Sep 2020 20:10:51 -0400 Subject: [PATCH 3/3] Update src/AutoBuild.jl Co-authored-by: Fredrik Ekre --- src/AutoBuild.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AutoBuild.jl b/src/AutoBuild.jl index 13f407d00..226538796 100644 --- a/src/AutoBuild.jl +++ b/src/AutoBuild.jl @@ -187,7 +187,7 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, while !isdir(joinpath(dir, "universes")) dir = dirname(dir) if dir == "/" - error("Could not find `universes` diretory in any parent of pwd") + error("Could not find `universes` directory in any parent of pwd") end end universe_path = joinpath(dir, "universes", universe_name)