diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f38832f6..3c465479d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,12 @@ on: - go.mod - go.sum +env: + # Used in tests to determine if certain tests should be skipped. + # Setting this ensures that they are *not* skipped and instead make sure CI + # is setup to be able to properly run all tests. + DALEC_CI: "1" + permissions: contents: read @@ -93,6 +99,8 @@ jobs: uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1 - name: download deps run: go mod download + - name: Setup QEMU + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - name: Run integaration tests run: go test -v -json ./test | go run ./cmd/test2json2gha - name: dump logs diff --git a/frontend/azlinux/azlinux3.go b/frontend/azlinux/azlinux3.go index c09b074a8..54665b7d2 100644 --- a/frontend/azlinux/azlinux3.go +++ b/frontend/azlinux/azlinux3.go @@ -3,7 +3,6 @@ package azlinux import ( "context" "encoding/json" - "path/filepath" "github.com/Azure/dalec" "github.com/moby/buildkit/client/llb" @@ -13,8 +12,7 @@ import ( ) const ( - AzLinux3TargetKey = "azlinux3" - tdnfCacheNameAzlinux3 = "azlinux3-tdnf-cache" + AzLinux3TargetKey = "azlinux3" // Azlinux3Ref is the image ref used for the base worker image Azlinux3Ref = "mcr.microsoft.com/azurelinux/base/core:3.0" @@ -49,6 +47,8 @@ func (w azlinux3) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S img := llb.Image(Azlinux3Ref, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...)) return img.Run( + w.Install([]string{"dnf"}, installWithConstraints(opts), tdnfOnly), + ).Run( w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, installWithConstraints(opts)), dalec.WithConstraints(opts...), ).Root(), nil @@ -57,7 +57,7 @@ func (w azlinux3) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S func (w azlinux3) Install(pkgs []string, opts ...installOpt) llb.RunOption { var cfg installConfig setInstallOptions(&cfg, opts) - return dalec.WithRunOptions(tdnfInstall(&cfg, "3.0", pkgs), w.tdnfCacheMount(cfg.root)) + return dalec.WithRunOptions(dnfInstall(&cfg, "3.0", pkgs, AzLinux3TargetKey)) } func (w azlinux3) BasePackages() []string { @@ -91,7 +91,3 @@ func (azlinux3) WorkerImageConfig(ctx context.Context, resolver llb.ImageMetaRes return &cfg, nil } - -func (azlinux3) tdnfCacheMount(root string) llb.RunOption { - return llb.AddMount(filepath.Join(root, tdnfCacheDir), llb.Scratch(), llb.AsPersistentCacheDir(tdnfCacheNameAzlinux3, llb.CacheMountLocked)) -} diff --git a/frontend/azlinux/handle_container.go b/frontend/azlinux/handle_container.go index e0863346a..4284774d8 100644 --- a/frontend/azlinux/handle_container.go +++ b/frontend/azlinux/handle_container.go @@ -25,7 +25,7 @@ func handleContainer(w worker) gwclient.BuildFunc { pg := dalec.ProgressGroup("Building " + targetKey + " container: " + spec.Name) - rpmDir, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg) + rpmDir, err := buildOutputRPM(ctx, w, client, spec, sOpt, targetKey, platform, pg) if err != nil { return nil, nil, fmt.Errorf("error creating rpm: %w", err) } @@ -35,7 +35,7 @@ func handleContainer(w worker) gwclient.BuildFunc { return nil, nil, err } - st, err := specToContainerLLB(w, spec, targetKey, rpmDir, rpms, sOpt, pg) + st, err := specToContainerLLB(w, spec, targetKey, rpmDir, rpms, sOpt, pg, dalec.WithPlatform(platform)) if err != nil { return nil, nil, err } diff --git a/frontend/azlinux/handle_depsonly.go b/frontend/azlinux/handle_depsonly.go index 068d516f1..a7499f8b8 100644 --- a/frontend/azlinux/handle_depsonly.go +++ b/frontend/azlinux/handle_depsonly.go @@ -21,7 +21,7 @@ func handleDepsOnly(w worker) gwclient.BuildFunc { return nil, nil, err } - baseImg, err := w.Base(sOpt, pg) + baseImg, err := w.Base(sOpt, pg, dalec.WithPlatform(platform)) if err != nil { return nil, nil, err } @@ -36,7 +36,7 @@ func handleDepsOnly(w worker) gwclient.BuildFunc { return nil, nil, err } - st, err := specToContainerLLB(w, spec, targetKey, rpmDir, files, sOpt, pg) + st, err := specToContainerLLB(w, spec, targetKey, rpmDir, files, sOpt, pg, dalec.WithPlatform(platform)) if err != nil { return nil, nil, err } diff --git a/frontend/azlinux/handle_rpm.go b/frontend/azlinux/handle_rpm.go index 87e89f545..2c3b45d48 100644 --- a/frontend/azlinux/handle_rpm.go +++ b/frontend/azlinux/handle_rpm.go @@ -4,13 +4,16 @@ import ( "context" "fmt" "path/filepath" + "strings" "github.com/Azure/dalec" "github.com/Azure/dalec/frontend" "github.com/Azure/dalec/frontend/rpm" + "github.com/containerd/platforms" "github.com/moby/buildkit/client/llb" gwclient "github.com/moby/buildkit/frontend/gateway/client" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) func handleRPM(w worker) gwclient.BuildFunc { @@ -26,7 +29,7 @@ func handleRPM(w worker) gwclient.BuildFunc { return nil, nil, err } - st, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg) + st, err := buildOutputRPM(ctx, w, client, spec, sOpt, targetKey, platform, pg) if err != nil { return nil, nil, err } @@ -52,11 +55,31 @@ func handleRPM(w worker) gwclient.BuildFunc { } } -// Creates and installs an rpm meta-package that requires the passed in deps as runtime-dependencies -func installBuildDepsPackage(target string, packageName string, w worker, deps map[string]dalec.PackageConstraints, installOpts ...installOpt) installFunc { +func platformFuzzyMatches(p *ocispecs.Platform) bool { + if p == nil { + return true + } + + // Note, this is intentionally not doing a strict match here + // (e.g. [platforms.OnlyStrict]) + // This is used to see if we can get some optimizations when building for a + // non-native platformm and in most cases the [platforms.Only] vector handles + // things like building armv7 on an arm64 machine, which should be able to run + // natively. + return platforms.Only(platforms.DefaultSpec()).Match(*p) +} + +func installBuildDeps(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { + deps := spec.GetBuildDeps(targetKey) + if len(deps) == 0 { + return func(in llb.State) llb.State { return in }, nil + } + + opts = append(opts, dalec.ProgressGroup("Install build deps")) + // depsOnly is a simple dalec spec that only includes build dependencies and their constraints depsOnly := dalec.Spec{ - Name: fmt.Sprintf("%s-build-dependencies", packageName), + Name: spec.Name + "-build-dependencies", Description: "Provides build dependencies for mariner2 and azlinux3", Version: "1.0", License: "Apache 2.0", @@ -66,72 +89,171 @@ func installBuildDepsPackage(target string, packageName string, w worker, deps m }, } - return func(ctx context.Context, client gwclient.Client, sOpt dalec.SourceOpts) (llb.RunOption, error) { - pg := dalec.ProgressGroup("Building container for build dependencies") + // create an RPM with just the build dependencies, using our same base worker + rpmDir, err := createRPM(w, sOpt, &depsOnly, targetKey, platform, opts...) + if err != nil { + return nil, err + } + + rpmMountDir := "/tmp/rpms" + pkg := []string{"/tmp/rpms/*/*.rpm"} - // create an RPM with just the build dependencies, using our same base worker - rpmDir, err := specToRpmLLB(ctx, w, client, &depsOnly, sOpt, target, pg) + if !platformFuzzyMatches(platform) { + base, err := w.Base(sOpt, opts...) if err != nil { return nil, err } - var opts []llb.ConstraintsOpt - opts = append(opts, dalec.ProgressGroup("Install build deps")) + return func(in llb.State) llb.State { + return base.Run( + w.Install( + pkg, + withMounts(llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS"))), + atRoot("/tmp/rootfs"), + withPlatform(platform), + ), + ).AddMount("/tmp/rootfs", in) + }, nil + } - rpmMountDir := "/tmp/rpms" + return func(in llb.State) llb.State { + return in.Run( + w.Install( + []string{"/tmp/rpms/*/*.rpm"}, + withMounts(llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS"))), + installWithConstraints(opts), + ), + dalec.WithConstraints(opts...), + ).Root() + }, nil +} - installOpts = append([]installOpt{ - noGPGCheck, - withMounts(llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS"))), - installWithConstraints(opts), - }, installOpts...) +func rpmWorker(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) { + base, err := w.Base(sOpt, append(opts, dalec.WithPlatform(platform))...) + if err != nil { + return llb.Scratch(), err + } - // install the built RPMs into the worker itself - return w.Install([]string{"/tmp/rpms/*/*.rpm"}, installOpts...), nil + installDeps, err := installBuildDeps(w, sOpt, spec, targetKey, platform, opts...) + if err != nil { + return llb.Scratch(), err } + + base = base.With(installDeps) + return base, nil } -func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { - deps := spec.GetBuildDeps(targetKey) - if len(deps) == 0 { - return func(in llb.State) llb.State { return in }, nil - } +func createBuildroot(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) { + opts = append(opts, dalec.ProgressGroup("Prepare rpm build root: "+spec.Name)) - sOpt, err := frontend.SourceOptFromClient(ctx, client) + // Always generate the build root using the native platform + // There is nothing it does that should require the requested target platform + native, err := w.Base(sOpt, opts...) if err != nil { - return nil, err + return llb.Scratch(), err } - opts = append(opts, dalec.ProgressGroup("Install build deps")) + if spec.HasGomods() { + // Since the spec has go mods in it, we need to make sure we have go + // installed in the image. + install, err := installBuildDeps(w, sOpt, spec, targetKey, nil, opts...) + if err != nil { + return llb.Scratch(), err + } - installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, deps, installWithConstraints(opts))(ctx, client, sOpt) - if err != nil { - return nil, err + native = native.With(install) } - return func(in llb.State) llb.State { - return in.Run(installOpt, dalec.WithConstraints(opts...)).Root() - }, nil + return rpm.SpecToBuildrootLLB(native, spec, sOpt, targetKey, opts...) } -func specToRpmLLB(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) { - base, err := w.Base(sOpt, opts...) - if err != nil { - return llb.Scratch(), err +func nativeGoMount(native llb.State, p *ocispecs.Platform) llb.RunOption { + const ( + gorootPath = "/usr/lib/golang" + goBinPath = "/usr/bin/go" + internalBinPath = "/tmp/internal/dalec/bin" + ) + + runOpts := []llb.RunOption{ + llb.AddMount(gorootPath, native, llb.SourcePath(gorootPath), llb.Readonly), + llb.AddEnv("GOARCH", p.Architecture), + dalec.RunOptFunc(func(ei *llb.ExecInfo) { + if p.Variant != "" { + switch p.Architecture { + case "arm": + // GOARM cannot have the `v` prefix that would be in the platform struct + llb.AddEnv("GOARM", strings.TrimPrefix(p.Variant, "v")).SetRunOption(ei) + case "amd64": + // Unlike GOARM, GOAMD64 must have the `v` prefix (Which should be + // present in the platform struct) + llb.AddEnv("GOAMD64", p.Variant).SetRunOption(ei) + default: + // go does not support any other special sub-architectures currently. + } + } + }), } - installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...) + return dalec.WithRunOptions(runOpts...) +} + +func hasGolangBuildDep(spec *dalec.Spec, targetKey string) bool { + deps := spec.GetBuildDeps(targetKey) + for pkg := range deps { + if pkg == "golang" || pkg == "msft-golang" { + return true + } + } + return false +} + +func platformOrDefault(p *ocispecs.Platform) ocispecs.Platform { + if p == nil { + return platforms.DefaultSpec() + } + return *p +} + +func createRPM(w worker, sOpt dalec.SourceOpts, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) { + br, err := createBuildroot(w, sOpt, spec, targetKey, opts...) if err != nil { - return llb.Scratch(), err + return llb.Scratch(), errors.Wrap(err, "error creating rpm build root") } - base = base.With(installOpt) - br, err := rpm.SpecToBuildrootLLB(base, spec, sOpt, targetKey, opts...) + base, err := rpmWorker(w, sOpt, spec, targetKey, platform, opts...) if err != nil { - return llb.Scratch(), err + return llb.Scratch(), nil + } + + var runOpts []llb.RunOption + if hasGolangBuildDep(spec, targetKey) { + if !platformFuzzyMatches(platform) { + native, err := rpmWorker(w, sOpt, spec, targetKey, nil, opts...) + if err != nil { + return llb.Scratch(), err + } + + runOpts = append(runOpts, nativeGoMount(native, platform)) + } + + const goCacheDir = "/tmp/dalec/internal/gocache" + runOpts = append(runOpts, llb.AddEnv("GOCACHE", goCacheDir)) + + // Unfortunately, go cannot invalidate caches for cgo (rather, cgo with 'include' directives). + // As such we need to include the platform in our cache key. + cacheKey := targetKey + "-golang-" + platforms.Format(platformOrDefault(platform)) + runOpts = append(runOpts, llb.AddMount(goCacheDir, llb.Scratch(), llb.AsPersistentCacheDir(cacheKey, llb.CacheMountShared))) } + specPath := filepath.Join("SPECS", spec.Name, spec.Name+".spec") - st := rpm.Build(br, base, specPath, opts...) + opts = append(opts, dalec.ProgressGroup("Create RPM: "+spec.Name)) + return rpm.Build(br, base, specPath, runOpts, opts...), nil +} +func buildOutputRPM(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) { + st, err := createRPM(w, sOpt, spec, targetKey, platform, opts...) + if err != nil { + return llb.Scratch(), err + } return frontend.MaybeSign(ctx, client, st, spec, targetKey, sOpt) } diff --git a/frontend/azlinux/handler.go b/frontend/azlinux/handler.go index dc2498f7e..68fc0e703 100644 --- a/frontend/azlinux/handler.go +++ b/frontend/azlinux/handler.go @@ -14,12 +14,6 @@ import ( ocispecs "github.com/opencontainers/image-spec/specs-go/v1" ) -const ( - tdnfCacheDir = "/var/cache/tdnf" -) - -type installFunc func(context.Context, gwclient.Client, dalec.SourceOpts) (llb.RunOption, error) - type worker interface { Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) Install(pkgs []string, opts ...installOpt) llb.RunOption @@ -62,12 +56,12 @@ func handleDebug(w worker) gwclient.BuildFunc { if err != nil { return nil, err } - return rpm.HandleDebug(getSpecWorker(ctx, w, client, sOpt))(ctx, client) + return rpm.HandleDebug(getSpecWorker(ctx, w, sOpt))(ctx, client) } } -func getSpecWorker(ctx context.Context, w worker, client gwclient.Client, sOpt dalec.SourceOpts) rpm.WorkerFunc { - return func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) { +func getSpecWorker(ctx context.Context, w worker, sOpt dalec.SourceOpts) rpm.WorkerFunc { + return func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) { st, err := w.Base(sOpt, opts...) if err != nil { return llb.Scratch(), err @@ -83,7 +77,7 @@ func getSpecWorker(ctx context.Context, w worker, client gwclient.Client, sOpt d return llb.Scratch(), errors.New("spec contains go modules but does not have golang in build deps") } - installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...) + installOpt, err := installBuildDeps(w, sOpt, spec, targetKey, platform, opts...) if err != nil { return llb.Scratch(), err } @@ -103,7 +97,7 @@ func handleBaseImg(w worker) gwclient.BuildFunc { return nil, nil, err } - st, err := w.Base(sOpt) + st, err := w.Base(sOpt, dalec.WithPlatform(platform)) if err != nil { return nil, nil, err } diff --git a/frontend/azlinux/install.go b/frontend/azlinux/install.go index c933e8335..25a3046ab 100644 --- a/frontend/azlinux/install.go +++ b/frontend/azlinux/install.go @@ -7,6 +7,7 @@ import ( "github.com/Azure/dalec" "github.com/moby/buildkit/client/llb" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" ) type installConfig struct { @@ -23,7 +24,14 @@ type installConfig struct { // Additional mounts to add to the tdnf install command (useful if installing RPMS which are mounted to a local directory) mounts []llb.RunOption + // Instructs the installer to install packages for the specified platform + platform *ocispecs.Platform + constraints []llb.ConstraintsOpt + + // This forces the use of tdnf + // Note this will almost certainly not work when platform is set. + tdnfOnly bool } type installOpt func(*installConfig) @@ -32,6 +40,10 @@ func noGPGCheck(cfg *installConfig) { cfg.noGPGCheck = true } +func tdnfOnly(cfg *installConfig) { + cfg.tdnfOnly = true +} + func withMounts(opts ...llb.RunOption) installOpt { return func(cfg *installConfig) { cfg.mounts = append(cfg.mounts, opts...) @@ -48,13 +60,19 @@ func atRoot(root string) installOpt { } } +func withPlatform(p *ocispecs.Platform) installOpt { + return func(cfg *installConfig) { + cfg.platform = p + } +} + func installWithConstraints(opts []llb.ConstraintsOpt) installOpt { return func(cfg *installConfig) { cfg.constraints = opts } } -func tdnfInstallFlags(cfg *installConfig) string { +func dnfInstallFlags(cfg *installConfig) string { var cmdOpts string if cfg.noGPGCheck { @@ -63,12 +81,30 @@ func tdnfInstallFlags(cfg *installConfig) string { if cfg.root != "" { cmdOpts += " --installroot=" + cfg.root - cmdOpts += " --setopt=reposdir=/etc/yum.repos.d" + cmdOpts += " --setopt reposdir=/etc/yum.repos.d" + } + + if cfg.platform != nil { + // cmdOpts += " --ignorearch=true" + cmdOpts += " --forcearch=" + ociArchToOS(cfg.platform) } return cmdOpts } +func ociArchToOS(p *ocispecs.Platform) string { + switch p.Architecture { + case "amd64": + return "x86_64" + case "arm64": + return "aarch64" + // azlinux only supports amd64 and arm64 + // We shouldn't need any other arches. + default: + return p.Architecture + } +} + func setInstallOptions(cfg *installConfig, opts []installOpt) { for _, o := range opts { o(cfg) @@ -111,9 +147,14 @@ rm -rf `+rpmdbDir+` const manifestSh = "manifest.sh" -func tdnfInstall(cfg *installConfig, relVer string, pkgs []string) llb.RunOption { - cmdFlags := tdnfInstallFlags(cfg) - cmdArgs := fmt.Sprintf("set -ex; tdnf install -y --refresh --releasever=%s %s %s", relVer, cmdFlags, strings.Join(pkgs, " ")) +func dnfInstall(cfg *installConfig, relVer string, pkgs []string, cachePrefix string) llb.RunOption { + cmdFlags := dnfInstallFlags(cfg) + + cmd := "dnf" + if cfg.tdnfOnly { + cmd = "tdnf" + } + cmdArgs := fmt.Sprintf("set -ex; %s install -y --refresh --releasever=%s %s %s", cmd, relVer, cmdFlags, strings.Join(pkgs, " ")) var runOpts []llb.RunOption @@ -128,6 +169,7 @@ func tdnfInstall(cfg *installConfig, relVer string, pkgs []string) llb.RunOption runOpts = append(runOpts, dalec.ShArgs(cmdArgs)) runOpts = append(runOpts, cfg.mounts...) + runOpts = append(runOpts, llb.AddMount("/var/cache/"+cmd, llb.Scratch(), llb.AsPersistentCacheDir(cachePrefix+"-"+cmd+"-"+"cache", llb.CacheMountLocked))) return dalec.WithRunOptions(runOpts...) } diff --git a/frontend/azlinux/mariner2.go b/frontend/azlinux/mariner2.go index 078d10336..0c2676959 100644 --- a/frontend/azlinux/mariner2.go +++ b/frontend/azlinux/mariner2.go @@ -3,7 +3,6 @@ package azlinux import ( "context" "encoding/json" - "path/filepath" "github.com/Azure/dalec" "github.com/moby/buildkit/client/llb" @@ -13,8 +12,7 @@ import ( ) const ( - Mariner2TargetKey = "mariner2" - tdnfCacheNameMariner2 = "mariner2-tdnf-cache" + Mariner2TargetKey = "mariner2" Mariner2Ref = "mcr.microsoft.com/cbl-mariner/base/core:2.0" Mariner2WorkerContextName = "dalec-mariner2-worker" @@ -46,6 +44,8 @@ func (w mariner2) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S } return base.Run( + w.Install([]string{"dnf"}, installWithConstraints(opts), tdnfOnly), + ).Run( w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, installWithConstraints(opts)), dalec.WithConstraints(opts...), ).Root(), nil @@ -54,7 +54,7 @@ func (w mariner2) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S func (w mariner2) Install(pkgs []string, opts ...installOpt) llb.RunOption { var cfg installConfig setInstallOptions(&cfg, opts) - return dalec.WithRunOptions(tdnfInstall(&cfg, "2.0", pkgs), w.tdnfCacheMount(cfg.root)) + return dalec.WithRunOptions(dnfInstall(&cfg, "2.0", pkgs, Mariner2TargetKey)) } func (w mariner2) BasePackages() []string { @@ -90,7 +90,3 @@ func (mariner2) WorkerImageConfig(ctx context.Context, resolver llb.ImageMetaRes return &cfg, nil } - -func (mariner2) tdnfCacheMount(root string) llb.RunOption { - return llb.AddMount(filepath.Join(root, tdnfCacheDir), llb.Scratch(), llb.AsPersistentCacheDir(tdnfCacheNameMariner2, llb.CacheMountLocked)) -} diff --git a/frontend/debug/handler.go b/frontend/debug/handler.go index cc64333d4..23958f844 100644 --- a/frontend/debug/handler.go +++ b/frontend/debug/handler.go @@ -25,6 +25,5 @@ func Handle(ctx context.Context, client gwclient.Client) (*gwclient.Result, erro Name: "gomods", Description: "Outputs all the gomodule dependencies for the spec", }) - return r.Handle(ctx, client) } diff --git a/frontend/gateway.go b/frontend/gateway.go index c8bf5c0bf..e6e0d0d10 100644 --- a/frontend/gateway.go +++ b/frontend/gateway.go @@ -17,10 +17,12 @@ import ( ) const ( - requestIDKey = "requestid" - dalecSubrequstForwardBuild = "dalec.forward.build" + // KeyRequestID is a key used in buildkit to performa subrequest + // This is exposed for convenience only. + KeyRequestID = "requestid" - gatewayFrontend = "gateway.v0" + dalecSubrequstForwardBuild = "dalec.forward.build" + gatewayFrontend = "gateway.v0" ) func getDockerfile(ctx context.Context, client gwclient.Client, build *dalec.SourceBuild, defPb *pb.Definition) ([]byte, error) { diff --git a/frontend/mux.go b/frontend/mux.go index 8df85341d..9950515ce 100644 --- a/frontend/mux.go +++ b/frontend/mux.go @@ -117,7 +117,7 @@ func (m *BuildMux) describe() (*gwclient.Result, error) { } func (m *BuildMux) handleSubrequest(ctx context.Context, client gwclient.Client, opts map[string]string) (*gwclient.Result, bool, error) { - switch opts[requestIDKey] { + switch opts[KeyRequestID] { case "": return nil, false, nil case subrequests.RequestSubrequestsDescribe: @@ -135,7 +135,7 @@ func (m *BuildMux) handleSubrequest(ctx context.Context, client gwclient.Client, res, err := handleDefaultPlatform() return res, true, err default: - return nil, false, errors.Errorf("unsupported subrequest %q", opts[requestIDKey]) + return nil, false, errors.Errorf("unsupported subrequest %q", opts[KeyRequestID]) } } @@ -369,7 +369,7 @@ func (m *BuildMux) Handle(ctx context.Context, client gwclient.Client) (_ *gwcli WithFields(logrus.Fields{ "handlers": maps.Keys(m.handlers), "target": opts[keyTarget], - "requestid": opts[requestIDKey], + "requestid": opts[KeyRequestID], "targetKey": GetTargetKey(client), })) @@ -403,7 +403,7 @@ func (m *BuildMux) Handle(ctx context.Context, client gwclient.Client) (_ *gwcli // If this request was a request to list targets, we need to modify the response a bit // Otherwise we can just return the result as is. - if opts[requestIDKey] == bktargets.SubrequestsTargetsDefinition.Name { + if opts[KeyRequestID] == bktargets.SubrequestsTargetsDefinition.Name { return m.fixupListResult(matched, res) } return res, nil diff --git a/frontend/rpm/handle_buildroot.go b/frontend/rpm/handle_buildroot.go index 8dee177b1..c47b91eac 100644 --- a/frontend/rpm/handle_buildroot.go +++ b/frontend/rpm/handle_buildroot.go @@ -11,7 +11,7 @@ import ( ocispecs "github.com/opencontainers/image-spec/specs-go/v1" ) -type WorkerFunc func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) +type WorkerFunc func(resolver llb.ImageMetaResolver, spec *dalec.Spec, targetKey string, platform *ocispecs.Platform, opts ...llb.ConstraintsOpt) (llb.State, error) func HandleBuildroot(wf WorkerFunc) gwclient.BuildFunc { return func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) { @@ -21,7 +21,9 @@ func HandleBuildroot(wf WorkerFunc) gwclient.BuildFunc { return nil, nil, err } - worker, err := wf(sOpt.Resolver, spec, targetKey) + // Note, we are not passing platform down here because everything should + // be able to work regardless of platform, so prefer the native platform. + worker, err := wf(sOpt.Resolver, spec, targetKey, platform) if err != nil { return nil, nil, err } diff --git a/frontend/rpm/handle_sources.go b/frontend/rpm/handle_sources.go index d46ddacc8..181166975 100644 --- a/frontend/rpm/handle_sources.go +++ b/frontend/rpm/handle_sources.go @@ -21,11 +21,13 @@ func HandleSources(wf WorkerFunc) gwclient.BuildFunc { return nil, nil, err } - worker, err := wf(sOpt.Resolver, spec, targetKey) + worker, err := wf(sOpt.Resolver, spec, targetKey, platform) if err != nil { return nil, nil, err } + // Note, we are not passing platform down here because everything should + // be able to work regardless of platform, so prefer the native platform. sources, err := Dalec2SourcesLLB(worker, spec, sOpt) if err != nil { return nil, nil, err diff --git a/frontend/rpm/rpmbuild.go b/frontend/rpm/rpmbuild.go index e46e8f9d5..290aad15b 100644 --- a/frontend/rpm/rpmbuild.go +++ b/frontend/rpm/rpmbuild.go @@ -16,7 +16,7 @@ import ( // It is expected to have rpmbuild and any other necessary build dependencies installed // // `specPath` is the path to the spec file to build relative to `topDir` -func Build(topDir, workerImg llb.State, specPath string, opts ...llb.ConstraintsOpt) llb.State { +func Build(topDir, workerImg llb.State, specPath string, runOpts []llb.RunOption, opts ...llb.ConstraintsOpt) llb.State { opts = append(opts, dalec.ProgressGroup("Build RPM")) return workerImg.Run( // some notes on these args: @@ -35,6 +35,11 @@ func Build(topDir, workerImg llb.State, specPath string, opts ...llb.Constraints llb.Dir("/build/top"), llb.Network(llb.NetModeNone), dalec.WithConstraints(opts...), + dalec.RunOptFunc(func(ei *llb.ExecInfo) { + for _, opt := range runOpts { + opt.SetRunOption(ei) + } + }), ). AddMount("/build/out", llb.Scratch()) } diff --git a/helpers.go b/helpers.go index 0d4832e37..5cca02e11 100644 --- a/helpers.go +++ b/helpers.go @@ -12,6 +12,7 @@ import ( "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/identity" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" ) var disableDiffMerge atomic.Bool @@ -413,3 +414,12 @@ func (s *Spec) GetPackageDeps(target string) *PackageDependencies { } return s.Dependencies } + +// WithPlatform sets the platform in the constraints opts +// This is similar to [llb.Platform] except this takes a pointer so you don't +// need to worry about dereferencing a potentially nil pointer. +func WithPlatform(p *ocispecs.Platform) llb.ConstraintsOpt { + return constraintsOptFunc(func(c *llb.Constraints) { + c.Platform = p + }) +} diff --git a/test/azlinux_test.go b/test/azlinux_test.go index f0e44372d..597ebf116 100644 --- a/test/azlinux_test.go +++ b/test/azlinux_test.go @@ -1,7 +1,9 @@ package test import ( + "bytes" "context" + "debug/elf" "errors" "fmt" "os" @@ -10,9 +12,16 @@ import ( "github.com/Azure/dalec" "github.com/Azure/dalec/frontend/azlinux" + "github.com/containerd/platforms" "github.com/moby/buildkit/client/llb" gwclient "github.com/moby/buildkit/frontend/gateway/client" moby_buildkit_v1_frontend "github.com/moby/buildkit/frontend/gateway/pb" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" +) + +var ( + linuxAmd64 = ocispecs.Platform{OS: "linux", Architecture: "amd64"} + linuxArm64 = ocispecs.Platform{OS: "linux", Architecture: "arm64"} ) var azlinuxConstraints = constraintsSymbols{ @@ -53,6 +62,7 @@ func TestMariner2(t *testing.T) { ID: "mariner", VersionID: "2.0", }, + SupportedPlatforms: platforms.Any(linuxAmd64, linuxArm64), }) } @@ -83,6 +93,7 @@ func TestAzlinux3(t *testing.T) { ID: "azurelinux", VersionID: "3.0", }, + SupportedPlatforms: platforms.Any(linuxAmd64, linuxArm64), }) } @@ -154,8 +165,9 @@ type testLinuxConfig struct { Units string Targets string } - Worker workerConfig - Release OSRelease + Worker workerConfig + Release OSRelease + SupportedPlatforms platforms.Matcher } type OSRelease struct { @@ -633,6 +645,7 @@ WantedBy=multi-user.target Dir: &dalec.SourceInlineDir{ Files: map[string]*dalec.SourceInlineFile{ + "foo.service": { Contents: ` [Unit] @@ -1174,6 +1187,11 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot }) }) + t.Run("platform", func(t *testing.T) { + ctx := startTestSpan(ctx, t) + testPlatforms(ctx, t, testConfig) + }) + t.Run("custom worker", func(t *testing.T) { t.Parallel() ctx := startTestSpan(baseCtx, t) @@ -1276,6 +1294,131 @@ func testCustomLinuxWorker(ctx context.Context, t *testing.T, targetCfg targetCo // Unfortunately it seems like there is an issue with the gateway client passing // in source policies. }) + +} + +func testPlatforms(ctx context.Context, t *testing.T, testConfig testLinuxConfig) { + if testConfig.SupportedPlatforms == nil { + t.Skip("List of supported platforms not provided for distro") + } + t.Run("build against different platform", func(t *testing.T) { + t.Parallel() + + ls, err := testEnv.Platforms(ctx) + if err != nil { + t.Fatal(err) + } + if len(ls) <= 1 { + t.Skipf("builder does not support multiple platforms: %s", platformsAsStringer(ls)) + } + + testEnv.RunTest(ctx, t, func(ctx context.Context, client gwclient.Client) { + p := readDefaultPlatform(ctx, t, client) + + matcher := platforms.OnlyStrict(p) + var testPlatform *ocispecs.Platform + for _, p2 := range ls { + // Get the first platform that is not the host platform that matches a supported distro platform + if !matcher.Match(p2) && testConfig.SupportedPlatforms.Match(p2) { + testPlatform = &p2 + break + } + } + + if testPlatform == nil { + msg := "could not find a platform suitable for testing, host platform: %s, available: %s" + ps := platformStringer(p) + workerPlatforms := platformsAsStringer(ls) + if os.Getenv("DALEC_CI") != "" { + t.Fatalf(msg, ps, workerPlatforms) + } + t.Skipf(msg, ps, workerPlatforms) + } + + spec := &dalec.Spec{ + Name: "test-platforms", + Version: "0.0.1", + Revision: "1", + Description: "Testing building on platform different from host platform", + License: "MIT", + Dependencies: &dalec.PackageDependencies{ + Build: map[string]dalec.PackageConstraints{ + "golang": {}, + }, + }, + Sources: map[string]dalec.Source{ + "src": { + Inline: &dalec.SourceInline{ + Dir: &dalec.SourceInlineDir{ + Files: map[string]*dalec.SourceInlineFile{ + "go.mod": { + Contents: "module test\n\ngo 1.21.6", + }, + "main.go": { + Contents: "package main\n\nfunc main() {}\n", + }, + }, + }, + }, + }, + }, + Build: dalec.ArtifactBuild{ + Steps: []dalec.BuildStep{ + {Command: "cd src; go build -o /tmp/test"}, + }, + }, + Artifacts: dalec.Artifacts{ + Binaries: map[string]dalec.ArtifactConfig{ + "/tmp/test": {}, + }, + }, + } + + tp := *testPlatform + req := newSolveRequest(withPlatform(tp), withSpec(ctx, t, spec), withBuildTarget(testConfig.Target.Container)) + res := solveT(ctx, t, client, req) + + imgPlatforms := readResultPlatforms(t, res) + if len(imgPlatforms) != 1 { + t.Fatal("expected image output to contain 1 platform") + } + + if !platforms.OnlyStrict(tp).Match(imgPlatforms[0]) { + t.Errorf("Expected image platform %q, got: %q", platformStringer(tp), platformStringer(imgPlatforms[0])) + } + + ref, err := res.SingleRef() + if err != nil { + t.Fatal(err) + } + if ref == nil { + t.Fatal("got empty reference -- most likely an empty (scratch) state was returned") + } + + // Read the ELF header so we can determine what the target architecture is. + dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{ + Filename: "/usr/bin/test", + }) + if err != nil { + t.Fatal(err) + } + + f, err := elf.NewFile(bytes.NewReader(dt)) + if err != nil { + t.Fatal(err) + } + + check := ocispecs.Platform{ + OS: "linux", + } + elfToPlatform(f, &check) + + if !platforms.OnlyStrict(*testPlatform).Match(check) { + t.Fatalf("output binary has unexpected platform, expected: %s, got: %s", platformStringer(*testPlatform), platformStringer(check)) + } + }) + }) + } func testPinnedBuildDeps(ctx context.Context, t *testing.T, cfg testLinuxConfig) { diff --git a/test/helpers_test.go b/test/helpers_test.go index 945498419..50ec95e7b 100644 --- a/test/helpers_test.go +++ b/test/helpers_test.go @@ -3,11 +3,14 @@ package test import ( "bytes" "context" + "debug/elf" + "encoding/binary" "encoding/json" "fmt" "io/fs" "path/filepath" "slices" + "strings" "testing" "github.com/Azure/dalec" @@ -20,6 +23,7 @@ import ( gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/frontend/subrequests/targets" "github.com/moby/buildkit/solver/pb" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/tonistiigi/fsutil/types" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/codes" @@ -197,6 +201,9 @@ func newSolveRequest(opts ...srOpt) gwclient.SolveRequest { func withPlatform(platform platforms.Platform) srOpt { return func(cfg *newSolveRequestConfig) { + if cfg.req.FrontendOpt == nil { + cfg.req.FrontendOpt = make(map[string]string) + } cfg.req.FrontendOpt["platform"] = platforms.Format(platform) } } @@ -221,21 +228,21 @@ func withSpec(ctx context.Context, t *testing.T, spec *dalec.Spec) srOpt { } } -func withBuildTarget(target string) srOpt { +func withSubrequest(id string) srOpt { return func(cfg *newSolveRequestConfig) { if cfg.req.FrontendOpt == nil { cfg.req.FrontendOpt = make(map[string]string) } - cfg.req.FrontendOpt["target"] = target + cfg.req.FrontendOpt["requestid"] = id } } -func withSubrequest(id string) srOpt { +func withBuildTarget(target string) srOpt { return func(cfg *newSolveRequestConfig) { if cfg.req.FrontendOpt == nil { cfg.req.FrontendOpt = make(map[string]string) } - cfg.req.FrontendOpt["requestid"] = id + cfg.req.FrontendOpt["target"] = target } } @@ -328,3 +335,60 @@ func readDefaultPlatform(ctx context.Context, t *testing.T, gwc gwclient.Client) assert.NilError(t, err) return p } + +func elfToPlatform(f *elf.File, target *ocispecs.Platform) { + switch f.Machine { + case elf.EM_X86_64: + target.Architecture = "amd64" + case elf.EM_ARM: + target.Architecture = "arm" + // TODO: subarch? + case elf.EM_AARCH64: + target.Architecture = "arm64" + case elf.EM_PPC64: + if f.ByteOrder == binary.LittleEndian { + target.Architecture = "ppc64le" + } + target.Architecture = "ppc64" + case elf.EM_S390: + target.Architecture = "s390x" + case elf.EM_RISCV: + target.Architecture = "riscv64" + } +} + +type platformsAsStringer []ocispecs.Platform + +func (ls platformsAsStringer) String() string { + collect := make([]string, 0, len(ls)) + for _, p := range ls { + collect = append(collect, platforms.Format(p)) + } + + return strings.Join(collect, ", ") +} + +func readResultPlatforms(t *testing.T, res *gwclient.Result) []ocispecs.Platform { + dt, ok := res.Metadata[exptypes.ExporterPlatformsKey] + if !ok { + return nil + } + + var pls exptypes.Platforms + if err := json.Unmarshal(dt, &pls); err != nil { + t.Fatal(err) + } + + out := make([]ocispecs.Platform, 0, len(pls.Platforms)) + for _, p := range pls.Platforms { + out = append(out, p.Platform) + } + + return out +} + +type platformStringer ocispecs.Platform + +func (p platformStringer) String() string { + return platforms.Format(ocispecs.Platform(p)) +} diff --git a/test/testenv/buildx.go b/test/testenv/buildx.go index 8d038beb6..3c2aec10b 100644 --- a/test/testenv/buildx.go +++ b/test/testenv/buildx.go @@ -15,9 +15,11 @@ import ( "sync" "testing" + "github.com/containerd/platforms" "github.com/moby/buildkit/client" gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/solver/pb" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" pkgerrors "github.com/pkg/errors" ) @@ -215,6 +217,31 @@ func (b *BuildxEnv) Buildkit(ctx context.Context) (*client.Client, error) { panic("unreachable: if you see this then this is a bug in the testenv bootstrap code") } +func (b *BuildxEnv) Platforms(ctx context.Context) ([]ocispecs.Platform, error) { + client, err := b.Buildkit(ctx) + if err != nil { + return nil, err + } + + workers, err := client.ListWorkers(ctx) + if err != nil { + return nil, err + } + + v := make(map[string]ocispecs.Platform) + for _, w := range workers { + for _, p := range w.Platforms { + v[platforms.Format(p)] = p + } + } + + out := make([]ocispecs.Platform, 0, len(v)) + for _, p := range v { + out = append(out, p) + } + return out, nil +} + type FrontendSpec struct { ID string Build gwclient.BuildFunc