From a8bcf64d4b7967bb0db74635fc89fb9f0ba05d65 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 4 Sep 2024 09:22:47 +0200 Subject: [PATCH 1/4] cmd/build: add new `-manifest-only` option This option allows build to just generate the manifest but not run `osbuild`. This is useful for the `otk` tests where a manifest for a specific distro needs to get generated to produce identical otk files. Note that we also have `gen-manifests` but that seems geared towards a different use-case so it's harder to use/change for this purpose. --- cmd/build/main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/build/main.go b/cmd/build/main.go index eedeb348e..1fc69c581 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -184,9 +184,11 @@ func filterRepos(repos []rpmmd.RepoConfig, typeName string) []rpmmd.RepoConfig { func run() error { // common args var outputDir, osbuildStore, rpmCacheRoot string + var manifestOnly bool flag.StringVar(&outputDir, "output", ".", "artifact output directory") flag.StringVar(&osbuildStore, "store", ".osbuild", "osbuild store for intermediate pipeline trees") flag.StringVar(&rpmCacheRoot, "rpmmd", "/tmp/rpmmd", "rpm metadata cache directory") + flag.BoolVar(&manifestOnly, "manifest-only", false, "only build the manifest, do not run osbuild") // osbuild checkpoint arg var checkpoints cmdutil.MultiValue @@ -252,7 +254,7 @@ func run() error { return fmt.Errorf("no repositories defined for %s/%s\n", distroName, archName) } - fmt.Printf("Generating manifest for %s: ", config.Name) + fmt.Printf("Generating manifest for %s\n", config.Name) mf, err := makeManifest(config, imgType, distribution, repos, archName, rpmCacheRoot) if err != nil { return err @@ -263,6 +265,9 @@ func run() error { if err := save(mf, manifestPath); err != nil { return err } + if manifestOnly { + return nil + } fmt.Printf("Building manifest: %s\n", manifestPath) From 0e30190376ec158708999080f099b1e18307af60 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 4 Sep 2024 09:30:07 +0200 Subject: [PATCH 2/4] cmd/build: make `-config` optional The `build` binary is perfectly happy with an empty build config so let's make this argument optional. Alternatively we could validate and enforce non-empty build configs if there is actually something in there that is important. --- cmd/build/main.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/cmd/build/main.go b/cmd/build/main.go index 1fc69c581..59f73887b 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -198,11 +198,11 @@ func run() error { var distroName, imgTypeName, configFile string flag.StringVar(&distroName, "distro", "", "distribution (required)") flag.StringVar(&imgTypeName, "type", "", "image type name (required)") - flag.StringVar(&configFile, "config", "", "build config file (required)") + flag.StringVar(&configFile, "config", "", "build config file") flag.Parse() - if distroName == "" || imgTypeName == "" || configFile == "" { + if distroName == "" || imgTypeName == "" { flag.Usage() os.Exit(1) } @@ -213,9 +213,14 @@ func run() error { } distroFac := distrofactory.NewDefault() - config, err := buildconfig.New(configFile) - if err != nil { - return err + var config *buildconfig.BuildConfig + if configFile != "" { + config, err = buildconfig.New(configFile) + if err != nil { + return err + } + } else { + config = &buildconfig.BuildConfig{} } if err := os.MkdirAll(outputDir, 0777); err != nil { @@ -254,7 +259,7 @@ func run() error { return fmt.Errorf("no repositories defined for %s/%s\n", distroName, archName) } - fmt.Printf("Generating manifest for %s\n", config.Name) + fmt.Printf("Generating manifest for %s", config.Name) mf, err := makeManifest(config, imgType, distribution, repos, archName, rpmCacheRoot) if err != nil { return err From 09db08dd1fca9a8b674ef29c186a758ff9f6fbc9 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 4 Sep 2024 09:43:39 +0200 Subject: [PATCH 3/4] cmd/build: add new `-manifest-path` flag to select manifest json This commit allows the user to select where the manifest json is writen. This is especially useful in combination with `--manifest-only` and is used by `otk` to generate reference images. --- cmd/build/main.go | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/cmd/build/main.go b/cmd/build/main.go index 59f73887b..c1217f232 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -144,6 +144,9 @@ func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d dist } func save(ms manifest.OSBuildManifest, fpath string) error { + if err := os.MkdirAll(filepath.Dir(fpath), 0777); err != nil { + return fmt.Errorf("cannot save manifest: %w", err) + } b, err := json.MarshalIndent(ms, "", " ") if err != nil { return fmt.Errorf("failed to marshal data for %q: %s\n", fpath, err.Error()) @@ -183,12 +186,13 @@ func filterRepos(repos []rpmmd.RepoConfig, typeName string) []rpmmd.RepoConfig { func run() error { // common args - var outputDir, osbuildStore, rpmCacheRoot string + var outputDir, osbuildStore, rpmCacheRoot, manifestPath string var manifestOnly bool flag.StringVar(&outputDir, "output", ".", "artifact output directory") flag.StringVar(&osbuildStore, "store", ".osbuild", "osbuild store for intermediate pipeline trees") flag.StringVar(&rpmCacheRoot, "rpmmd", "/tmp/rpmmd", "rpm metadata cache directory") flag.BoolVar(&manifestOnly, "manifest-only", false, "only build the manifest, do not run osbuild") + flag.StringVar(&manifestPath, "manifest-path", "", "write manifest to the give path") // osbuild checkpoint arg var checkpoints cmdutil.MultiValue @@ -223,10 +227,6 @@ func run() error { config = &buildconfig.BuildConfig{} } - if err := os.MkdirAll(outputDir, 0777); err != nil { - return fmt.Errorf("failed to create target directory: %s", err.Error()) - } - distribution := distroFac.GetDistro(distroName) if distribution == nil { return fmt.Errorf("invalid or unsupported distribution: %q", distroName) @@ -238,12 +238,6 @@ func run() error { return fmt.Errorf("invalid arch name %q for distro %q: %s\n", archName, distroName, err.Error()) } - buildName := fmt.Sprintf("%s-%s-%s-%s", u(distroName), u(archName), u(imgTypeName), u(config.Name)) - buildDir := filepath.Join(outputDir, buildName) - if err := os.MkdirAll(buildDir, 0777); err != nil { - return fmt.Errorf("failed to create target directory: %s", err.Error()) - } - imgType, err := arch.GetImageType(imgTypeName) if err != nil { return fmt.Errorf("invalid image type %q for distro %q and arch %q: %s\n", imgTypeName, distroName, archName, err.Error()) @@ -266,7 +260,11 @@ func run() error { } fmt.Print("DONE\n") - manifestPath := filepath.Join(buildDir, "manifest.json") + buildName := fmt.Sprintf("%s-%s-%s-%s", u(distroName), u(archName), u(imgTypeName), u(config.Name)) + if manifestPath == "" { + buildDir := filepath.Join(outputDir, buildName) + manifestPath = filepath.Join(buildDir, "manifest.json") + } if err := save(mf, manifestPath); err != nil { return err } @@ -277,6 +275,14 @@ func run() error { fmt.Printf("Building manifest: %s\n", manifestPath) jobOutput := filepath.Join(outputDir, buildName) + if err := os.MkdirAll(jobOutput, 0777); err != nil { + return fmt.Errorf("failed to create output directory: %w", err) + } + buildDir := filepath.Join(outputDir, buildName) + if err := os.MkdirAll(buildDir, 0777); err != nil { + return fmt.Errorf("failed to create target directory: %w", err) + } + _, err = osbuild.RunOSBuild(mf, osbuildStore, jobOutput, imgType.Exports(), checkpoints, nil, false, os.Stderr) if err != nil { return err From 95e39d3bd0136808a11b903f7d4d894c7279bf4e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 4 Sep 2024 09:55:31 +0200 Subject: [PATCH 4/4] cmd/build: add `-target-arch` flag This flag allows to set the target architecture for the build. This is most useful for `-manifest-only` and the intended use-case is generating reference images for `otk`. But it could also be used with `qemu-user` and should work equally well as `bib` (maybe better as the "regular" distro code tends to use less of the modern system calls that tend to be problematic like `openat2`). --- cmd/build/main.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/build/main.go b/cmd/build/main.go index c1217f232..b64cef3ec 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -199,10 +199,11 @@ func run() error { flag.Var(&checkpoints, "checkpoints", "comma-separated list of pipeline names to checkpoint (passed to osbuild --checkpoint)") // image selection args - var distroName, imgTypeName, configFile string + var distroName, imgTypeName, configFile, targetArch string flag.StringVar(&distroName, "distro", "", "distribution (required)") flag.StringVar(&imgTypeName, "type", "", "image type name (required)") flag.StringVar(&configFile, "config", "", "build config file") + flag.StringVar(&targetArch, "target-arch", "", "target architecture to use, mostly for -manifest-only") flag.Parse() @@ -232,7 +233,12 @@ func run() error { return fmt.Errorf("invalid or unsupported distribution: %q", distroName) } - archName := arch.Current().String() + var archName string + if targetArch != "" { + archName = targetArch + } else { + archName = arch.Current().String() + } arch, err := distribution.GetArch(archName) if err != nil { return fmt.Errorf("invalid arch name %q for distro %q: %s\n", archName, distroName, err.Error())