azlinux/mariner: Use provided platform for outputs

Before this change, the mariner/azl targets were ignoring the client provided platform for the produced aritifacts. With this change the platform is set on images so it will rely on binfmt_misc to execute those images correctly. There's probably some optimizations that can be made here to run certain tasks on the native platform, some thoughts: - Use native (host-arch) golang to download go modules - Use native (host-arch) tdnf to download/install non-native packages onto the target build environment. This does *not* add support for cross compilation (run x86 code to generate, e.g., arm64 code). Signed-off-by: Brian Goff <[email protected]>
Azure · Jul 25, 2024 · 3f0d0ec · 3f0d0ec
1 parent 8c1d340
commit 3f0d0ec
Show file tree

Hide file tree

Showing 13 changed files with 316 additions and 17 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,6 +31,12 @@ on:
       - go.mod
       - go.sum
 
+env:
+  # Used in tests to determine if certain tests should be skipped.
+  # Setting this ensures that they are *not* skipped and instead make sure CI
+  # is setup to be able to properly run all tests.
+  DALEC_CI: "1"
+
 permissions:
   contents: read
 
@@ -93,6 +99,8 @@ jobs:
         uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v3.4.0
       - name: download deps
         run: go mod download
+      - name: Setup QEMU
+        run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
       - name: Run integaration tests
         run: go test -v -json ./test | go run ./cmd/test2json2gha
       - name: dump logs

diff --git a/frontend/azlinux/handle_container.go b/frontend/azlinux/handle_container.go
@@ -25,7 +25,7 @@ func handleContainer(w worker) gwclient.BuildFunc {
 
 			pg := dalec.ProgressGroup("Building " + targetKey + " container: " + spec.Name)
 
-			rpmDir, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg)
+			rpmDir, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg, dalec.WithPlatform(platform))
 			if err != nil {
 				return nil, nil, fmt.Errorf("error creating rpm: %w", err)
 			}
@@ -35,7 +35,7 @@ func handleContainer(w worker) gwclient.BuildFunc {
 				return nil, nil, err
 			}
 
-			st, err := specToContainerLLB(w, spec, targetKey, rpmDir, rpms, sOpt, pg)
+			st, err := specToContainerLLB(w, spec, targetKey, rpmDir, rpms, sOpt, pg, dalec.WithPlatform(platform))
 			if err != nil {
 				return nil, nil, err
 			}

diff --git a/frontend/azlinux/handle_depsonly.go b/frontend/azlinux/handle_depsonly.go
@@ -21,7 +21,7 @@ func handleDepsOnly(w worker) gwclient.BuildFunc {
 				return nil, nil, err
 			}
 
-			baseImg, err := w.Base(sOpt, pg)
+			baseImg, err := w.Base(sOpt, pg, dalec.WithPlatform(platform))
 			if err != nil {
 				return nil, nil, err
 			}

diff --git a/frontend/azlinux/handle_rpm.go b/frontend/azlinux/handle_rpm.go
@@ -26,7 +26,7 @@ func handleRPM(w worker) gwclient.BuildFunc {
 				return nil, nil, err
 			}
 
-			st, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg)
+			st, err := specToRpmLLB(ctx, w, client, spec, sOpt, targetKey, pg, dalec.WithPlatform(platform))
 			if err != nil {
 				return nil, nil, err
 			}

diff --git a/frontend/debug/handler.go b/frontend/debug/handler.go
@@ -25,6 +25,5 @@ func Handle(ctx context.Context, client gwclient.Client) (*gwclient.Result, erro
 		Name:        "gomods",
 		Description: "Outputs all the gomodule dependencies for the spec",
 	})
-
 	return r.Handle(ctx, client)
 }
diff --git a/frontend/gateway.go b/frontend/gateway.go
@@ -17,10 +17,12 @@ import (
 )
 
 const (
-	requestIDKey               = "requestid"
-	dalecSubrequstForwardBuild = "dalec.forward.build"
+	// KeyRequestID is a key used in buildkit to performa subrequest
+	// This is exposed for convenience only.
+	KeyRequestID = "requestid"
 
-	gatewayFrontend = "gateway.v0"
+	dalecSubrequstForwardBuild = "dalec.forward.build"
+	gatewayFrontend            = "gateway.v0"
 )
 
 func getDockerfile(ctx context.Context, client gwclient.Client, build *dalec.SourceBuild, defPb *pb.Definition) ([]byte, error) {

diff --git a/frontend/mux.go b/frontend/mux.go
@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/Azure/dalec"
+	"github.com/containerd/containerd/platforms"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/frontend/dockerui"
 	gwclient "github.com/moby/buildkit/frontend/gateway/client"
@@ -21,6 +22,11 @@ import (
 	"golang.org/x/exp/maps"
 )
 
+const (
+	// KeySubrequestHostPlatform is a dalec specific subrequest for determining the platform of the worker.
+	KeyHostPlatform = "dalec.debug.platform"
+)
+
 // BuildMux implements a buildkit BuildFunc via its Handle method. With a
 // BuildMux you register routes with mux.Add("someKey", SomeHandler,
 // optionalTargetInfo).
@@ -108,7 +114,7 @@ func (m *BuildMux) describe() (*gwclient.Result, error) {
 }
 
 func (m *BuildMux) handleSubrequest(ctx context.Context, client gwclient.Client, opts map[string]string) (*gwclient.Result, bool, error) {
-	switch opts[requestIDKey] {
+	switch opts[KeyRequestID] {
 	case "":
 		return nil, false, nil
 	case subrequests.RequestSubrequestsDescribe:
@@ -117,13 +123,31 @@ func (m *BuildMux) handleSubrequest(ctx context.Context, client gwclient.Client,
 	case bktargets.SubrequestsTargetsDefinition.Name:
 		res, err := m.list(ctx, client, opts[keyTarget])
 		return res, true, err
+	case KeyHostPlatform:
+		res, err := m.hostPlatform()
+		return res, true, err
 	case keyTopLevelTarget:
 		return nil, false, nil
 	default:
-		return nil, false, errors.Errorf("unsupported subrequest %q", opts[requestIDKey])
+		return nil, false, errors.Errorf("unsupported subrequest %q", opts[KeyRequestID])
 	}
 }
 
+func (m *BuildMux) hostPlatform() (*gwclient.Result, error) {
+	res := gwclient.NewResult()
+
+	p := platforms.DefaultSpec()
+	dt, err := json.Marshal(p)
+	if err != nil {
+		return nil, err
+	}
+
+	res.AddMeta("result.json", dt)
+	res.AddMeta("result.txt", []byte(platforms.Format(p)))
+
+	return res, nil
+}
+
 func (m *BuildMux) loadSpec(ctx context.Context, client gwclient.Client) (*dalec.Spec, error) {
 	if m.spec != nil {
 		return m.spec, nil
@@ -299,7 +323,7 @@ func (m *BuildMux) Handle(ctx context.Context, client gwclient.Client) (_ *gwcli
 		WithFields(logrus.Fields{
 			"handlers":  maps.Keys(m.handlers),
 			"target":    opts[keyTarget],
-			"requestid": opts[requestIDKey],
+			"requestid": opts[KeyRequestID],
 			"targetKey": GetTargetKey(client),
 		}))
 
@@ -333,7 +357,7 @@ func (m *BuildMux) Handle(ctx context.Context, client gwclient.Client) (_ *gwcli
 
 	// If this request was a request to list targets, we need to modify the response a bit
 	// Otherwise we can just return the result as is.
-	if opts[requestIDKey] == bktargets.SubrequestsTargetsDefinition.Name {
+	if opts[KeyRequestID] == bktargets.SubrequestsTargetsDefinition.Name {
 		return m.fixupListResult(matched, res)
 	}
 	return res, nil

diff --git a/go.mod b/go.mod
@@ -6,6 +6,7 @@ toolchain go1.21.0
 
 require (
 	github.com/containerd/containerd v1.7.13
+	github.com/containerd/platforms v0.2.1
 	github.com/goccy/go-yaml v1.11.3
 	github.com/google/go-cmp v0.6.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510

diff --git a/go.sum b/go.sum
@@ -48,6 +48,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/containerd/nydus-snapshotter v0.13.7 h1:x7DHvGnzJOu1ZPwPYkeOPk5MjZZYbdddygEjaSDoFTk=
 github.com/containerd/nydus-snapshotter v0.13.7/go.mod h1:VPVKQ3jmHFIcUIV2yiQ1kImZuBFS3GXDohKs9mRABVE=
+github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
+github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
 github.com/containerd/stargz-snapshotter v0.15.1 h1:fpsP4kf/Z4n2EYnU0WT8ZCE3eiKDwikDhL6VwxIlgeA=
 github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU=
 github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk=

diff --git a/helpers.go b/helpers.go
@@ -12,6 +12,7 @@ import (
 
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/identity"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 var disableDiffMerge atomic.Bool
@@ -399,3 +400,12 @@ func SortedMapValues[T any](m map[string]T) []T {
 
 	return out
 }
+
+// WithPlatform sets the platform in the constraints opts
+// This is similar to [llb.Platform] except this takes a pointer so you don't
+// need to worry about dereferencing a potentially nil pointer.
+func WithPlatform(p *ocispecs.Platform) llb.ConstraintsOpt {
+	return constraintsOptFunc(func(c *llb.Constraints) {
+		c.Platform = p
+	})
+}