diff --git a/frontend/build.go b/frontend/build.go index 146133b8..b851a709 100644 --- a/frontend/build.go +++ b/frontend/build.go @@ -72,15 +72,7 @@ func fillPlatformArgs(prefix string, args map[string]string, platform ocispecs.P type PlatformBuildFunc func(ctx context.Context, client gwclient.Client, platform *ocispecs.Platform, spec *dalec.Spec, targetKey string) (gwclient.Reference, *dalec.DockerImageSpec, error) -// BuildWithPlatform is a helper function to build a spec with a given platform -// It takes care of looping through each tarrget platform and executing the build with the platform args substituted in the spec. -// This also deals with the docker-style multi-platform output. -func BuildWithPlatform(ctx context.Context, client gwclient.Client, f PlatformBuildFunc) (*gwclient.Result, error) { - dc, err := dockerui.NewClient(client) - if err != nil { - return nil, err - } - +func BuildWithPlatformFromUIClient(ctx context.Context, client gwclient.Client, dc *dockerui.Client, f PlatformBuildFunc) (*gwclient.Result, error) { rb, err := dc.Build(ctx, func(ctx context.Context, platform *ocispecs.Platform, idx int) (gwclient.Reference, *dalec.DockerImageSpec, *dalec.DockerImageSpec, error) { spec, err := LoadSpec(ctx, dc, platform) if err != nil { @@ -101,6 +93,17 @@ func BuildWithPlatform(ctx context.Context, client gwclient.Client, f PlatformBu return rb.Finalize() } +// BuildWithPlatform is a helper function to build a spec with a given platform +// It takes care of looping through each tarrget platform and executing the build with the platform args substituted in the spec. +// This also deals with the docker-style multi-platform output. +func BuildWithPlatform(ctx context.Context, client gwclient.Client, f PlatformBuildFunc) (*gwclient.Result, error) { + dc, err := dockerui.NewClient(client) + if err != nil { + return nil, err + } + return BuildWithPlatformFromUIClient(ctx, client, dc, f) +} + // GetBaseImage returns an image that first checks if the client provided the // image in the build context matching the image ref. // diff --git a/frontend/gateway.go b/frontend/gateway.go index 862afb67..415e4a8c 100644 --- a/frontend/gateway.go +++ b/frontend/gateway.go @@ -101,12 +101,7 @@ func GetBuildArg(client gwclient.Client, k string) (string, bool) { return "", false } -func SourceOptFromClient(ctx context.Context, c gwclient.Client) (dalec.SourceOpts, error) { - dc, err := dockerui.NewClient(c) - if err != nil { - return dalec.SourceOpts{}, err - } - +func SourceOptFromUIClient(ctx context.Context, c gwclient.Client, dc *dockerui.Client) dalec.SourceOpts { return dalec.SourceOpts{ Resolver: c, Forward: ForwarderFromClient(ctx, c), @@ -125,7 +120,15 @@ func SourceOptFromClient(ctx context.Context, c gwclient.Client) (dalec.SourceOp } return st, nil }, - }, nil + } +} + +func SourceOptFromClient(ctx context.Context, c gwclient.Client) (dalec.SourceOpts, error) { + dc, err := dockerui.NewClient(c) + if err != nil { + return dalec.SourceOpts{}, err + } + return SourceOptFromUIClient(ctx, c, dc), nil } var ( diff --git a/frontend/windows/dockerui.go b/frontend/windows/dockerui.go new file mode 100644 index 00000000..8781604b --- /dev/null +++ b/frontend/windows/dockerui.go @@ -0,0 +1,123 @@ +package windows + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/containerd/platforms" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/frontend/dockerui" + gwclient "github.com/moby/buildkit/frontend/gateway/client" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +// This is a copy of dockerui.Client.Build +// It has one modification: Instead of `platforms.Format` it uses `platforms.FormatAll` +// The value returned from this function is used as a map key to store build +// result references. +// When `platforms.Format` is used, the `OSVersion` field is not taken into account +// which means we end up overwriting map keys when there are multiple windows +// platform images being output but with different OSVerions. +// platforms.FormatAll takes OSVersion into account. +func dcBuild(ctx context.Context, bc *dockerui.Client, fn dockerui.BuildFunc) (*resultBuilder, error) { + res := gwclient.NewResult() + + targets := make([]*ocispecs.Platform, 0, len(bc.TargetPlatforms)) + for _, p := range bc.TargetPlatforms { + p := p + targets = append(targets, &p) + } + if len(targets) == 0 { + targets = append(targets, nil) + } + expPlatforms := &exptypes.Platforms{ + Platforms: make([]exptypes.Platform, len(targets)), + } + + eg, ctx := errgroup.WithContext(ctx) + + for i, tp := range targets { + i, tp := i, tp + eg.Go(func() error { + ref, img, baseImg, err := fn(ctx, tp, i) + if err != nil { + return err + } + + config, err := json.Marshal(img) + if err != nil { + return errors.Wrapf(err, "failed to marshal image config") + } + + var baseConfig []byte + if baseImg != nil { + baseConfig, err = json.Marshal(baseImg) + if err != nil { + return errors.Wrapf(err, "failed to marshal source image config") + } + } + + p := platforms.DefaultSpec() + if tp != nil { + p = *tp + } + + // in certain conditions we allow input platform to be extended from base image + if p.OS == "windows" && img.OS == p.OS { + if p.OSVersion == "" && img.OSVersion != "" { + p.OSVersion = img.OSVersion + } + if p.OSFeatures == nil && len(img.OSFeatures) > 0 { + p.OSFeatures = append([]string{}, img.OSFeatures...) + } + } + + p = platforms.Normalize(p) + k := platforms.FormatAll(p) + + if bc.MultiPlatformRequested { + res.AddRef(k, ref) + res.AddMeta(fmt.Sprintf("%s/%s", exptypes.ExporterImageConfigKey, k), config) + if len(baseConfig) > 0 { + res.AddMeta(fmt.Sprintf("%s/%s", exptypes.ExporterImageBaseConfigKey, k), baseConfig) + } + } else { + res.SetRef(ref) + res.AddMeta(exptypes.ExporterImageConfigKey, config) + if len(baseConfig) > 0 { + res.AddMeta(exptypes.ExporterImageBaseConfigKey, baseConfig) + } + } + expPlatforms.Platforms[i] = exptypes.Platform{ + ID: k, + Platform: p, + } + return nil + }) + } + if err := eg.Wait(); err != nil { + return nil, err + } + return &resultBuilder{ + Result: res, + expPlatforms: expPlatforms, + }, nil +} + +type resultBuilder struct { + *gwclient.Result + expPlatforms *exptypes.Platforms +} + +func (rb *resultBuilder) Finalize() (*gwclient.Result, error) { + dt, err := json.Marshal(rb.expPlatforms) + if err != nil { + return nil, err + } + rb.AddMeta(exptypes.ExporterPlatformsKey, dt) + + return rb.Result, nil +} diff --git a/frontend/windows/handle_container.go b/frontend/windows/handle_container.go index bd88a231..fe7be4f5 100644 --- a/frontend/windows/handle_container.go +++ b/frontend/windows/handle_container.go @@ -4,16 +4,22 @@ import ( "context" "encoding/json" "fmt" + "io/fs" "path" "runtime" + "sync" "github.com/Azure/dalec" "github.com/Azure/dalec/frontend" + "github.com/Azure/dalec/frontend/pkg/bkfs" + "github.com/containerd/platforms" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb/sourceresolver" + "github.com/moby/buildkit/frontend/dockerui" gwclient "github.com/moby/buildkit/frontend/gateway/client" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) const ( @@ -32,35 +38,196 @@ var ( } ) +type ImageBases struct { + Refs []string `json:"refs,omitempty" yaml:"refs,omitempty"` +} + +func (ib *ImageBases) getRefs() []string { + if ib == nil { + return nil + } + return ib.Refs +} + +func (ib *ImageBases) len() int { + if ib == nil { + return 0 + } + return len(ib.Refs) +} + +var defaultImageBases = &ImageBases{ + Refs: []string{ + defaultBaseImage, + "mcr.microsoft.com/windows/nanoserver:ltsc2022", + }, +} + +func getImageBases(ctx context.Context, client gwclient.Client, sOpt dalec.SourceOpts) (*ImageBases, error) { + const ( + argBasesPathKey = "build-arg:DALEC_WINDOWSCROSS_BASES_PATH" + argBasesContextKey = "build-arg:DALEC_WINDOWSCROSS_BASES_CONTEXT" + ) + + bOpts := client.BuildOpts().Opts + p := bOpts[argBasesPathKey] + if p == "" { + return nil, nil + } + + src := dalec.Source{ + Context: &dalec.SourceContext{Name: "context"}, + Path: p, + } + + if name := bOpts[argBasesContextKey]; name != "" { + src.Context.Name = name + } + + pg := dalec.ProgressGroup("Determine base images") + st, err := src.AsMount("src", sOpt, pg) + if err != nil { + return nil, err + } + + root, err := bkfs.FromState(ctx, &st, client, pg) + if err != nil { + return nil, err + } + + dt, err := fs.ReadFile(root, p) + if err != nil { + return nil, err + } + + var bases ImageBases + + if err := json.Unmarshal(dt, &bases); err != nil { + return nil, err + } + return &bases, nil +} + func handleContainer(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) { - return frontend.BuildWithPlatform(ctx, client, func(ctx context.Context, client gwclient.Client, platform *ocispecs.Platform, spec *dalec.Spec, targetKey string) (gwclient.Reference, *dalec.DockerImageSpec, error) { - sOpt, err := frontend.SourceOptFromClient(ctx, client) + dc, err := dockerui.NewClient(client) + if err != nil { + return nil, err + } + + if len(dc.TargetPlatforms) > 1 { + return nil, fmt.Errorf("multi-platform output is not supported") + } + + sOpt := frontend.SourceOptFromUIClient(ctx, client, dc) + + bases, err := getImageBases(ctx, client, sOpt) + if err != nil { + return nil, err + } + + refs := bases.getRefs() + if len(refs) == 0 { + refs = append(refs, defaultBaseImage) + } + + eg, grpCtx := errgroup.WithContext(ctx) + cfgs := make([][]byte, len(refs)) + targets := make([]ocispecs.Platform, len(cfgs)) + + basePlatform := defaultPlatform + if len(dc.TargetPlatforms) > 0 { + basePlatform = dc.TargetPlatforms[0] + } + + for idx, ref := range refs { + idx := idx + ref := ref + eg.Go(func() error { + _, _, dt, err := client.ResolveImageConfig(grpCtx, ref, sourceresolver.Opt{ + Platform: &basePlatform, + ImageOpt: &sourceresolver.ResolveImageOpt{ + ResolveMode: dc.ImageResolveMode.String(), + }, + }) + + if err != nil { + return err + } + + var cfg dalec.DockerImageSpec + if err := json.Unmarshal(dt, &cfg); err != nil { + return errors.Wrapf(err, "image config for %s", ref) + } + + cfgs[idx] = dt + targets[idx] = cfg.Platform + + return nil + }) + } + + if err := eg.Wait(); err != nil { + return nil, err + } + + seen := make(map[string]struct{}) + for _, p := range targets { + s := platforms.FormatAll(p) + if _, ok := seen[s]; ok { + return nil, fmt.Errorf("mutiple base images provided with the same platform value") + } + seen[s] = struct{}{} + } + + dc.TargetPlatforms = targets + if len(targets) > 1 { + dc.MultiPlatformRequested = true + } + targetKey := frontend.GetTargetKey(client) + + warnBaseOverride := sync.OnceFunc(func() { + frontend.Warn(ctx, client, llb.Scratch(), "Base image defined in spec overwritten by base images context") + }) + + getBaseRef := func(idx int, spec *dalec.Spec) string { + baseRef := refs[idx] + + updated := dalec.GetBaseOutputImage(spec, targetKey) + if updated == "" { + return baseRef + } + + if bases.len() == 0 { + return updated + } + + warnBaseOverride() + return baseRef + } + + rb, err := dcBuild(ctx, dc, func(ctx context.Context, platform *ocispecs.Platform, idx int) (ref gwclient.Reference, retCfg, retBaseCfg *dalec.DockerImageSpec, retErr error) { + spec, err := frontend.LoadSpec(ctx, dc, platform) if err != nil { - return nil, nil, err + return nil, nil, nil, err } if err := validateRuntimeDeps(spec, targetKey); err != nil { - return nil, nil, fmt.Errorf("error validating windows spec: %w", err) + return nil, nil, nil, fmt.Errorf("error validating windows spec: %w", err) } pg := dalec.ProgressGroup("Build windows container: " + spec.Name) worker, err := distroConfig.Worker(sOpt, pg) if err != nil { - return nil, nil, err + return nil, nil, nil, err } bin, err := buildBinaries(ctx, spec, worker, client, sOpt, targetKey) if err != nil { - return nil, nil, fmt.Errorf("unable to build binary %w", err) - } - - baseImgName := getBaseOutputImage(spec, targetKey, defaultBaseImage) - - if platform == nil { - platform = &defaultPlatform + return nil, nil, nil, fmt.Errorf("unable to build binary %w", err) } - baseImage := llb.Image(baseImgName, llb.Platform(*platform)) + baseRef := getBaseRef(idx, spec) + baseImage := llb.Image(baseRef, llb.Platform(*platform)) out := baseImage. File(llb.Copy(bin, "/", windowsSystemDir)). @@ -68,40 +235,36 @@ func handleContainer(ctx context.Context, client gwclient.Client) (*gwclient.Res def, err := out.Marshal(ctx) if err != nil { - return nil, nil, err + return nil, nil, nil, err } res, err := client.Solve(ctx, gwclient.SolveRequest{ Definition: def.ToPB(), }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - imgRef := dalec.GetBaseOutputImage(spec, targetKey) - if imgRef == "" { - imgRef = defaultBaseImage - } - - _, _, dt, err := client.ResolveImageConfig(ctx, imgRef, sourceresolver.Opt{ - Platform: platform, - }) - if err != nil { - return nil, nil, errors.Wrap(err, "could not resolve base image config") + var baseCfg dalec.DockerImageSpec + if err := json.Unmarshal(cfgs[idx], &baseCfg); err != nil { + return nil, nil, nil, errors.Wrap(err, "error unmarshalling base image config") } + // Get a copy of the cfg so we can modify it var img dalec.DockerImageSpec - if err := json.Unmarshal(dt, &img); err != nil { - return nil, nil, errors.Wrap(err, "error unmarshalling base image config") + if err := json.Unmarshal(cfgs[idx], &img); err != nil { + return nil, nil, nil, errors.Wrap(err, "error unmarshalling base image config") } if err := dalec.BuildImageConfig(spec, targetKey, &img); err != nil { - return nil, nil, errors.Wrap(err, "error creating image config") + return nil, nil, nil, errors.Wrap(err, "error creating image config") } - ref, err := res.SingleRef() - return ref, &img, err + ref, err = res.SingleRef() + return ref, &img, &baseCfg, err }) + + return rb.Finalize() } func copySymlinks(post *dalec.PostInstall) llb.StateOption { @@ -124,13 +287,4 @@ func copySymlinks(post *dalec.PostInstall) llb.StateOption { return s } - -} - -func getBaseOutputImage(spec *dalec.Spec, target, defaultBase string) string { - baseRef := defaultBase - if spec.Targets[target].Image != nil && spec.Targets[target].Image.Base != "" { - baseRef = spec.Targets[target].Image.Base - } - return baseRef } diff --git a/load.go b/load.go index 79236ee2..d8cf7bb4 100644 --- a/load.go +++ b/load.go @@ -30,6 +30,10 @@ func knownArg(key string) bool { return true case "DALEC_SKIP_TESTS": return true + case "DALEC_WINDOWSCROSS_BASES_PATH": + return true + case "DALEC_WINDOWSCROSS_BASES_CONTEXT": + return true } return platformArg(key) diff --git a/test/windows_test.go b/test/windows_test.go index e5c6fae1..ce0095f8 100644 --- a/test/windows_test.go +++ b/test/windows_test.go @@ -2,6 +2,7 @@ package test import ( "context" + "encoding/json" "errors" "fmt" "testing" @@ -9,11 +10,16 @@ import ( "github.com/Azure/dalec" "github.com/Azure/dalec/frontend/ubuntu" "github.com/Azure/dalec/frontend/windows" + "github.com/containerd/platforms" "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/client/llb/sourceresolver" + "github.com/moby/buildkit/exporter/containerimage/exptypes" gwclient "github.com/moby/buildkit/frontend/gateway/client" moby_buildkit_v1_frontend "github.com/moby/buildkit/frontend/gateway/pb" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/exp/maps" + "gotest.tools/v3/assert" + "gotest.tools/v3/assert/cmp" ) var windowsAmd64 = ocispecs.Platform{OS: "windows", Architecture: "amd64"} @@ -184,6 +190,9 @@ func testWindows(ctx context.Context, t *testing.T, tcfg targetConfig) { }) }) t.Run("container", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(ctx, t) + spec := dalec.Spec{ Name: "test-container-build", Version: "0.0.1", @@ -316,17 +325,8 @@ echo "$BAR" > bar.txt }, } - testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { - sr := newSolveRequest(withSpec(ctx, t, &spec), withBuildTarget(tcfg.Container), withWindowsAmd64) - sr.Evaluate = true - res := solveT(ctx, t, gwc, sr) - - ref, err := res.SingleRef() - if err != nil { - t.Fatal(err) - } - - post := spec.GetImagePost("windowscross") + post := spec.GetImagePost("windowscross") + validateSymlinks := func(ctx context.Context, t *testing.T, ref gwclient.Reference) { for srcPath, l := range post.Symlinks { b1, err := ref.ReadFile(ctx, gwclient.ReadRequest{ Filename: srcPath, @@ -352,6 +352,115 @@ echo "$BAR" > bar.txt } } } + } + + t.Run("single-image", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(ctx, t) + + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { + sr := newSolveRequest(withSpec(ctx, t, &spec), withBuildTarget(tcfg.Container), withWindowsAmd64) + sr.Evaluate = true + res := solveT(ctx, t, gwc, sr) + + ref, err := res.SingleRef() + if err != nil { + t.Fatal(err) + } + validateSymlinks(ctx, t, ref) + }) + }) + + t.Run("multi-image", func(t *testing.T) { + t.Parallel() + ctx := startTestSpan(ctx, t) + + testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) { + + bases := windows.ImageBases{ + Refs: []string{ + "mcr.microsoft.com/windows/nanoserver:ltsc2022", + "mcr.microsoft.com/windows/nanoserver:1809", + }, + } + + dt, err := json.Marshal(bases) + assert.NilError(t, err) + + st := llb.Scratch().File( + llb.Mkfile("bases.json", 0o644, dt), + ) + sr := newSolveRequest( + withSpec(ctx, t, &spec), + withBuildTarget(tcfg.Container), + withWindowsAmd64, + withBuildContext(ctx, t, "multi-image", st), + withBuildArg("DALEC_WINDOWSCROSS_BASES_PATH", "bases.json"), + withBuildArg("DALEC_WINDOWSCROSS_BASES_CONTEXT", "multi-image"), + ) + sr.Evaluate = true + res := solveT(ctx, t, gwc, sr) + + var metaPlatforms exptypes.Platforms + err = json.Unmarshal(res.Metadata["refs.platforms"], &metaPlatforms) + assert.NilError(t, err) + assert.Assert(t, cmp.Len(metaPlatforms.Platforms, 2)) + + // Go through each of the base images we requested and resolve + // them so we can get the platform info + // Then validate that the platform for the base image matches the platform + // in the result platforms. + for i, ref := range bases.Refs { + actual := metaPlatforms.Platforms[i] + + _, _, dt, err = gwc.ResolveImageConfig(ctx, ref, sourceresolver.Opt{ + Platform: &windowsAmd64, + }) + + var cfg dalec.DockerImageSpec + assert.NilError(t, json.Unmarshal(dt, &cfg)) + assert.Check(t, cmp.Equal(cfg.OS, actual.Platform.OS)) + assert.Check(t, cmp.Equal(cfg.Architecture, actual.Platform.Architecture)) + assert.Check(t, cmp.Equal(cfg.OSVersion, actual.Platform.OSVersion)) + } + + // NOTE: we are not using `res.SingleRef` because we requested multiple + // refs which would cause an error in this case. + // Instead we need to look at res.Refs + assert.Assert(t, cmp.Len(res.Refs, len(metaPlatforms.Platforms))) + + for _, p := range metaPlatforms.Platforms { + ref, ok := res.Refs[platforms.FormatAll(p.Platform)] + assert.Assert(t, ok, "unepxected ref keys: %s", maps.Keys(res.Refs)) + validateSymlinks(ctx, t, ref) + } + + // This should fail since the bases have the same platform + bases = windows.ImageBases{ + Refs: []string{ + "mcr.microsoft.com/windows/nanoserver:ltsc2022", + "mcr.microsoft.com/windows/nanoserver:ltsc2022-amd64", + }, + } + + dt, err = json.Marshal(bases) + assert.NilError(t, err) + + st = llb.Scratch().File( + llb.Mkfile("bases.json", 0o644, dt), + ) + sr = newSolveRequest( + withSpec(ctx, t, &spec), + withBuildTarget(tcfg.Container), + withWindowsAmd64, + withBuildContext(ctx, t, "multi-image", st), + withBuildArg("DALEC_WINDOWSCROSS_BASES_PATH", "bases.json"), + withBuildArg("DALEC_WINDOWSCROSS_BASES_CONTEXT", "multi-image"), + ) + sr.Evaluate = true + _, err = gwc.Solve(ctx, sr) + assert.ErrorContains(t, err, "mutiple base images provided with the same") + }) }) }) diff --git a/website/docs/targets.md b/website/docs/targets.md index ed58eb95..ee53d1df 100644 --- a/website/docs/targets.md +++ b/website/docs/targets.md @@ -144,3 +144,42 @@ This works the same way in the `azlinux3`: i. `--build-context mcr.microsoft.com/azurelinux/base/core:3.0=` 2. A build context named `dalec-mariner2-worker` i. `--build-context dalec-azlinux3-worker=` + +#### Windows + +For Windows containers, typically the container image OS needs to match +the Windows host OS. + +You can use DALEC to create a single multi-platform image with the different +Windows versions you want to use. +Normally you would specify a single base image in the DALEC spec's image config, +however this is not sufficient to accomplish this task. + +With DALEC you can pass in a build-arg `DALEC_WINDOWSCROSS_BASES_PATH` the value +of which should be the path to a file containing json with the following +structure to the `windowscross/container` build target: + + +```json +{ + "refs": [ + "mcr.microsoft.com/windows/nanoserver:1809", + "mcr.microsoft.com/windows/nanoserver:ltsc2022" + ] +} +``` + +:::note +Values in the "refs" field can be any Windows image. + +You can provide any number of images here, however each image must have a +different value for the `os.version` field in the image config's platform. +If there are images with the same platform values the build wil fail. +::: + +You can also provide this file in a named build context, but you must still +specifiy the above mentioned build-arg so that DALEC knows how to find the file +in that named context. +You can tell DALEC to use a named context by providing the name in a build-arg +`DALEC_WINDOWSCROSS_BASES_CONTEXT` +