diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index fbedf2ea7a2..b89adbb85c0 100755 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -8,6 +8,9 @@ #: "=/work/manifest.toml", #: "=/work/repo.zip", #: "=/work/repo.zip.sha256.txt", +#: "=/work/helios.json", +#: "=/work/incorporation.p5m", +#: "=/work/incorporation.p5p", #: "%/work/*.log", #: ] #: access_repos = [ @@ -42,6 +45,21 @@ #: name = "repo.zip.sha256.txt" #: from_output = "/work/repo.zip.sha256.txt" #: +#: [[publish]] +#: series = "rot-all" +#: name = "helios.json" +#: from_output = "/work/helios.json" +#: +#: [[publish]] +#: series = "rot-all" +#: name = "incorporation.p5m" +#: from_output = "/work/incorporation.p5m" +#: +#: [[publish]] +#: series = "rot-all" +#: name = "incorporation.p5m" +#: from_output = "/work/incorporation.p5p" +#: set -o errexit set -o pipefail @@ -73,4 +91,4 @@ esac pfexec zfs create -p "rpool/images/$USER/host" pfexec zfs create -p "rpool/images/$USER/recovery" -cargo xtask releng --output-dir /work +cargo xtask releng --output-dir /work --mkincorp diff --git a/Cargo.lock b/Cargo.lock index 613122af7d1..21f98094f5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7675,6 +7675,7 @@ dependencies = [ "reqwest", "semver 1.0.26", "serde", + "serde_json", "sha2", "shell-words", "slog", diff --git a/dev-tools/releng/Cargo.toml b/dev-tools/releng/Cargo.toml index 78af644f4a8..1cff65995fc 100644 --- a/dev-tools/releng/Cargo.toml +++ b/dev-tools/releng/Cargo.toml @@ -20,6 +20,7 @@ omicron-zone-package.workspace = true reqwest.workspace = true semver.workspace = true serde.workspace = true +serde_json.workspace = true sha2.workspace = true shell-words.workspace = true slog.workspace = true diff --git a/dev-tools/releng/src/helios.rs b/dev-tools/releng/src/helios.rs new file mode 100644 index 00000000000..44e3706fb5b --- /dev/null +++ b/dev-tools/releng/src/helios.rs @@ -0,0 +1,174 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::Context; +use anyhow::Result; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use fs_err::tokio as fs; +use fs_err::tokio::File; +use serde::Deserialize; +use slog::Logger; +use tokio::io::AsyncWriteExt; +use tokio::io::BufWriter; + +use crate::HELIOS_REPO; +use crate::Jobs; +use crate::cmd::Command; + +pub const INCORP_NAME: &str = + "consolidation/oxide/omicron-release-incorporation"; +const MANIFEST_PATH: &str = "incorporation.p5m"; +const REPO_PATH: &str = "incorporation"; +pub const ARCHIVE_PATH: &str = "incorporation.p5p"; + +pub const PUBLISHER: &str = "helios-dev"; + +pub(crate) enum Action { + Generate { version: String }, + Passthru { version: String }, +} + +pub(crate) async fn push_incorporation_jobs( + jobs: &mut Jobs, + logger: &Logger, + output_dir: &Utf8Path, + action: Action, +) -> Result<()> { + let manifest_path = output_dir.join(MANIFEST_PATH); + let repo_path = output_dir.join(REPO_PATH); + let archive_path = output_dir.join(ARCHIVE_PATH); + + fs::remove_dir_all(&repo_path).await.or_else(ignore_not_found)?; + fs::remove_file(&archive_path).await.or_else(ignore_not_found)?; + + match action { + Action::Generate { version } => { + jobs.push( + "incorp-manifest", + generate_incorporation_manifest( + logger.clone(), + manifest_path.clone(), + version, + ), + ); + } + Action::Passthru { version } => { + jobs.push( + "incorp-manifest", + passthru_incorporation_manifest( + logger.clone(), + manifest_path.clone(), + version, + ), + ); + } + } + + jobs.push_command( + "incorp-fmt", + Command::new("pkgfmt").args(["-u", "-f", "v2", manifest_path.as_str()]), + ) + .after("incorp-manifest"); + + jobs.push_command( + "incorp-create", + Command::new("pkgrepo").args(["create", repo_path.as_str()]), + ); + + let path_args = ["-s", repo_path.as_str()]; + jobs.push_command( + "incorp-publisher", + Command::new("pkgrepo") + .arg("add-publisher") + .args(&path_args) + .arg(PUBLISHER), + ) + .after("incorp-create"); + + jobs.push_command( + "incorp-pkgsend", + Command::new("pkgsend") + .arg("publish") + .args(&path_args) + .arg(manifest_path), + ) + .after("incorp-fmt") + .after("incorp-publisher"); + + jobs.push_command( + "helios-incorp", + Command::new("pkgrecv") + .args(path_args) + .args(["-a", "-d", archive_path.as_str()]) + .args(["-m", "latest", "-v", "*"]), + ) + .after("incorp-pkgsend"); + + Ok(()) +} + +async fn generate_incorporation_manifest( + logger: Logger, + path: Utf8PathBuf, + version: String, +) -> Result<()> { + #[derive(Deserialize, PartialEq, Eq, PartialOrd, Ord)] + struct Package { + fmri: String, + } + + let mut manifest = BufWriter::new(File::create(path).await?); + let preamble = format!( + r#"set name=pkg.fmri value=pkg://{PUBLISHER}/{INCORP_NAME}@{version},5.11 +set name=pkg.summary value="Incorporation to constrain software delivered in Omicron Release V{version} images" +set name=info.classification value="org.opensolaris.category.2008:Meta Packages/Incorporations" +set name=variant.opensolaris.zone value=global value=nonglobal +"# + ); + manifest.write_all(preamble.as_bytes()).await?; + + let stdout = Command::new("pkg") + .args(["list", "-g", HELIOS_REPO, "-F", "json"]) + .args(["-o", "fmri", "*@latest"]) + .ensure_stdout(&logger) + .await?; + let packages: Vec = serde_json::from_str(&stdout) + .context("failed to parse pkgrepo output")?; + let prefix = format!("pkg://{PUBLISHER}/"); + for package in packages { + let Some(partial) = package.fmri.strip_prefix(&prefix) else { + continue; + }; + let Some((package, _)) = partial.split_once('@') else { + continue; + }; + if package == INCORP_NAME || package == "driver/network/opte" { + continue; + } + let line = format!("depend type=incorporate fmri=pkg:/{partial}\n"); + manifest.write_all(line.as_bytes()).await?; + } + + manifest.shutdown().await?; + Ok(()) +} + +async fn passthru_incorporation_manifest( + logger: Logger, + path: Utf8PathBuf, + version: String, +) -> Result<()> { + let stdout = Command::new("pkgrepo") + .args(["contents", "-m", "-s", HELIOS_REPO]) + .arg(format!("pkg://{PUBLISHER}/{INCORP_NAME}@{version},5.11")) + .ensure_stdout(&logger) + .await?; + fs::write(&path, stdout).await?; + Ok(()) +} + +fn ignore_not_found(err: std::io::Error) -> Result<(), std::io::Error> { + if err.kind() == std::io::ErrorKind::NotFound { Ok(()) } else { Err(err) } +} diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index f209bc3a68c..52f33e4ff24 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -3,10 +3,13 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. mod cmd; +mod helios; mod hubris; mod job; mod tuf; +use std::collections::BTreeMap; +use std::path::PathBuf; use std::sync::Arc; use std::sync::LazyLock; use std::time::Duration; @@ -15,6 +18,7 @@ use std::time::Instant; use anyhow::Context; use anyhow::Result; use anyhow::bail; +use camino::Utf8Path; use camino::Utf8PathBuf; use chrono::Utc; use clap::Parser; @@ -152,6 +156,19 @@ struct Args { /// Extra manifest to be merged with the rest of the repo #[clap(long)] extra_manifest: Option, + + /// Extra helios-dev origin to be passed along to helios-build + #[clap(long)] + extra_origin: Option, + + /// Create and use an `omicron-ci-incorporation` package during the image + /// build. The incorporation can then be reused during branching to pin + /// packages in the image to the same version. + /// + /// This option does nothing if the `[helios]` table is present in + /// tools/pins.toml. + #[clap(long, conflicts_with("helios_local"))] + mkincorp: bool, } impl Args { @@ -276,22 +293,19 @@ async fn main() -> Result<()> { // Ensure the Helios checkout exists. If the directory exists and is // non-empty, check that the commit is correct. - if args.helios_dir.exists() + let helios_commit = if args.helios_dir.exists() && fs::read_dir(&args.helios_dir).await?.next_entry().await?.is_some() { + let helios_commit = git_resolve_commit( + &args.git_bin, + &args.helios_dir, + "HEAD", + &logger, + ) + .await?; if !args.ignore_helios_origin { if let Some(helios) = &pins.helios { - let stdout = Command::new(&args.git_bin) - .arg("-C") - .arg(&args.helios_dir) - .args(["rev-parse", "HEAD"]) - .ensure_stdout(&logger) - .await?; - let line = stdout - .lines() - .next() - .context("git-rev-parse output was empty")?; - if line != helios.commit { + if helios_commit != helios.commit { error!( logger, "helios checkout at {0} is not at pinned commit {1}; \ @@ -307,24 +321,20 @@ async fn main() -> Result<()> { Command::new(&args.git_bin) .arg("-C") .arg(&args.helios_dir) - .args([ - "fetch", - "--no-write-fetch-head", - "origin", - "master", - ]) + // HEAD in a remote repository refers to the default + // branch, even if the default branch is renamed. + // `--no-write-fetch-head` avoids modifying FETCH_HEAD. + .args(["fetch", "--no-write-fetch-head", "origin", "HEAD"]) .ensure_success(&logger) .await?; - let stdout = Command::new(&args.git_bin) - .arg("-C") - .arg(&args.helios_dir) - .args(["rev-parse", "HEAD", "origin/master"]) - .ensure_stdout(&logger) - .await?; - let mut lines = stdout.lines(); - let first = - lines.next().context("git-rev-parse output was empty")?; - if !lines.all(|line| line == first) { + let upstream_commit = git_resolve_commit( + &args.git_bin, + &args.helios_dir, + "origin/HEAD", + &logger, + ) + .await?; + if helios_commit != upstream_commit { error!( logger, "helios checkout at {0} is out-of-date; run \ @@ -336,6 +346,7 @@ async fn main() -> Result<()> { } } } + helios_commit } else { info!(logger, "cloning helios to {}", args.helios_dir); Command::new(&args.git_bin) @@ -354,8 +365,12 @@ async fn main() -> Result<()> { .args(["checkout", &helios.commit]) .ensure_success(&logger) .await?; + helios.commit.clone() + } else { + git_resolve_commit(&args.git_bin, &args.helios_dir, "HEAD", &logger) + .await? } - } + }; // Check that the omicron1 brand is installed if !Command::new("pkg") @@ -404,14 +419,6 @@ async fn main() -> Result<()> { .context("failed to create temporary directory")?; let mut jobs = Jobs::new(&logger, permits.clone(), &args.output_dir); - // Record the branch and commit of helios.git in the output - Command::new(&args.git_bin) - .arg("-C") - .arg(&args.helios_dir) - .args(["status", "--branch", "--porcelain=2"]) - .ensure_success(&logger) - .await?; - jobs.push_command( "helios-setup", Command::new("ptime") @@ -428,6 +435,42 @@ async fn main() -> Result<()> { .env_remove("RUSTUP_TOOLCHAIN"), ); + // Record the commit of helios.git and everything helios-setup cloned + { + let git_bin = args.git_bin.clone(); + let output_dir = args.output_dir.clone(); + let helios_dir = args.helios_dir.clone(); + let logger = logger.clone(); + jobs.push("helios-record", async move { + let projects_dir = helios_dir.join("projects"); + let mut projects = BTreeMap::new(); + let mut read_dir = fs::read_dir(&projects_dir).await?; + while let Some(entry) = read_dir.next_entry().await? { + let name = + Utf8PathBuf::try_from(PathBuf::from(entry.file_name()))?; + let commit = git_resolve_commit( + &git_bin, + &projects_dir.join(&name), + "HEAD", + &logger, + ) + .await?; + projects.insert(name, commit); + } + + let json = format!( + "{:#}\n", + serde_json::json!({ + "helios": helios_commit, + "projects": projects, + }) + ); + fs::write(output_dir.join("helios.json"), json).await?; + Ok(()) + }) + .after("helios-setup"); + } + let omicron_package = if let Some(path) = &args.omicron_package_bin { // omicron-package is provided, so don't build it. jobs.push("omicron-package", std::future::ready(Ok(()))); @@ -477,6 +520,24 @@ async fn main() -> Result<()> { }}; } + let incorp_version = format!("{}.0.0.0", version.major); + if args.mkincorp { + let action = if let Some(helios) = &pins.helios { + helios::Action::Passthru { version: helios.incorporation.clone() } + } else { + helios::Action::Generate { version: incorp_version.clone() } + }; + helios::push_incorporation_jobs( + &mut jobs, + &logger, + &args.output_dir, + action, + ) + .await?; + } else { + jobs.push("helios-incorp", std::future::ready(Ok(()))); + } + for target in [Target::Host, Target::Recovery] { let artifacts_path = target.artifacts_path(&args); @@ -572,12 +633,37 @@ async fn main() -> Result<()> { .env_remove("CARGO") .env_remove("RUSTUP_TOOLCHAIN"); + if let Some(extra_origin) = &args.extra_origin { + image_cmd = image_cmd + .arg("-p") + .arg(format!("{}={extra_origin}", helios::PUBLISHER)); + } + if let Some(helios) = &pins.helios { image_cmd = image_cmd.arg("-F").arg(format!( - "extra_packages+=\ - /consolidation/oxide/omicron-release-incorporation@{}", + "extra_packages+=/{}@{}", + helios::INCORP_NAME, helios.incorporation )); + } else if args.mkincorp { + image_cmd = image_cmd + .arg("-F") + .arg(format!( + "extra_packages+=/{}@{incorp_version}", + helios::INCORP_NAME + )) + .arg("-p") + .arg(format!( + "{}=file://{}", + helios::PUBLISHER, + args.output_dir + .canonicalize_utf8() + .with_context(|| format!( + "failed to canonicalize {}", + args.output_dir + ))? + .join(helios::ARCHIVE_PATH) + )); } if !args.helios_local { @@ -589,6 +675,7 @@ async fn main() -> Result<()> { // helios-build experiment-image jobs.push_command(format!("{}-image", target), image_cmd) .after("helios-setup") + .after("helios-incorp") .after(format!("{}-proto", target)); } // Build the recovery target after we build the host target. Only one @@ -751,7 +838,7 @@ async fn build_proto_area( let cloned_path = path.clone(); let cloned_package_dir = package_dir.to_owned(); tokio::task::spawn_blocking(move || -> Result<()> { - let mut archive = tar::Archive::new(std::fs::File::open( + let mut archive = tar::Archive::new(fs_err::File::open( cloned_package_dir .join(package_name.as_str()) .with_extension("tar"), @@ -797,3 +884,18 @@ async fn host_add_root_profile(host_proto_root: Utf8PathBuf) -> Result<()> { ).await?; Ok(()) } + +async fn git_resolve_commit( + git_bin: &Utf8Path, + repo: &Utf8Path, + what: &str, + logger: &Logger, +) -> Result { + Ok(Command::new(git_bin) + .args(["-C", repo.as_str(), "rev-parse", "--verify"]) + .arg(format!("{what}^{{commit}}")) + .ensure_stdout(&logger) + .await? + .trim() + .to_owned()) +}