Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Support OS version in platform string #5614

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion client/llb/imagemetaresolver/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func (imr *imageMetaResolver) ResolveImageConfig(ctx context.Context, ref string

func (imr *imageMetaResolver) key(ref string, platform *ocispecs.Platform) string {
if platform != nil {
ref += platforms.Format(*platform)
ref += platforms.FormatAll(*platform)
}
return ref
}
Expand Down
2 changes: 1 addition & 1 deletion exporter/containerimage/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (ag AnnotationsGroup) Platform(p *ocispecs.Platform) *Annotations {

ps := []string{""}
if p != nil {
ps = append(ps, platforms.Format(*p))
ps = append(ps, platforms.FormatAll(*p))
}

for _, a := range ag {
Expand Down
2 changes: 1 addition & 1 deletion exporter/containerimage/exptypes/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (k AnnotationKey) PlatformString() string {
if k.Platform == nil {
return ""
}
return platforms.Format(*k.Platform)
return platforms.FormatAll(*k.Platform)
}

func AnnotationIndexKey(key string) string {
Expand Down
2 changes: 1 addition & 1 deletion exporter/containerimage/exptypes/parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func ParsePlatforms(meta map[string][]byte) (Platforms, error) {
}
}
p = platforms.Normalize(p)
pk := platforms.Format(p)
pk := platforms.FormatAll(p)
ps := Platforms{
Platforms: []Platform{{ID: pk, Platform: p}},
}
Expand Down
13 changes: 7 additions & 6 deletions exporter/verifier/platforms.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,14 @@ func CheckInvalidPlatforms[T comparable](ctx context.Context, res *result.Result
})
}
p = platforms.Normalize(p)
_, ok := reqMap[platforms.Format(p)]
formatted := platforms.FormatAll(p)
_, ok := reqMap[formatted]
if ok {
warnings = append(warnings, client.VertexWarning{
Short: []byte(fmt.Sprintf("Duplicate platform result requested %q", v)),
})
}
reqMap[platforms.Format(p)] = struct{}{}
reqMap[formatted] = struct{}{}
reqList = append(reqList, exptypes.Platform{Platform: p})
}

Expand All @@ -62,9 +63,9 @@ func CheckInvalidPlatforms[T comparable](ctx context.Context, res *result.Result

if len(reqMap) == 1 && len(ps.Platforms) == 1 {
pp := platforms.Normalize(ps.Platforms[0].Platform)
if _, ok := reqMap[platforms.Format(pp)]; !ok {
if _, ok := reqMap[platforms.FormatAll(pp)]; !ok {
return []client.VertexWarning{{
Short: []byte(fmt.Sprintf("Requested platform %q does not match result platform %q", req.Platforms[0], platforms.Format(pp))),
Short: []byte(fmt.Sprintf("Requested platform %q does not match result platform %q", req.Platforms[0], platforms.FormatAll(pp))),
}}, nil
}
return nil, nil
Expand All @@ -81,7 +82,7 @@ func CheckInvalidPlatforms[T comparable](ctx context.Context, res *result.Result
if !mismatch {
for _, p := range ps.Platforms {
pp := platforms.Normalize(p.Platform)
if _, ok := reqMap[platforms.Format(pp)]; !ok {
if _, ok := reqMap[platforms.FormatAll(pp)]; !ok {
mismatch = true
break
}
Expand All @@ -100,7 +101,7 @@ func CheckInvalidPlatforms[T comparable](ctx context.Context, res *result.Result
func platformsString(ps []exptypes.Platform) string {
var ss []string
for _, p := range ps {
ss = append(ss, platforms.Format(platforms.Normalize(p.Platform)))
ss = append(ss, platforms.FormatAll(platforms.Normalize(p.Platform)))
}
sort.Strings(ss)
return strings.Join(ss, ",")
Expand Down
2 changes: 1 addition & 1 deletion frontend/dockerfile/builder/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) {
if platform != nil {
p = *platform
}
scanTargets.Store(platforms.Format(platforms.Normalize(p)), scanTarget)
scanTargets.Store(platforms.FormatAll(platforms.Normalize(p)), scanTarget)

return ref, img, baseImg, nil
})
Expand Down
10 changes: 5 additions & 5 deletions frontend/dockerfile/dockerfile2llb/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -519,7 +519,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
if reachable {
prefix := "["
if opt.MultiPlatformRequested && platform != nil {
prefix += platforms.Format(*platform) + " "
prefix += platforms.FormatAll(*platform) + " "
}
prefix += "internal]"
mutRef, dgst, dt, err := metaResolver.ResolveImageConfig(ctx, d.stage.BaseName, sourceresolver.Opt{
Expand Down Expand Up @@ -2102,7 +2102,7 @@ func prefixCommand(ds *dispatchState, str string, prefixPlatform bool, platform
}
out := "["
if prefixPlatform && platform != nil {
out += platforms.Format(*platform) + formatTargetPlatform(*platform, platformFromEnv(env)) + " "
out += platforms.FormatAll(*platform) + formatTargetPlatform(*platform, platformFromEnv(env)) + " "
}
if ds.stageName != "" {
out += ds.stageName + " "
Expand Down Expand Up @@ -2136,7 +2136,7 @@ func formatTargetPlatform(base ocispecs.Platform, target *ocispecs.Platform) str
return "->" + archVariant
}
if p.OS != base.OS {
return "->" + platforms.Format(p)
return "->" + platforms.FormatAll(p)
}
return ""
}
Expand Down Expand Up @@ -2483,8 +2483,8 @@ func wrapSuggestAny(err error, keys map[string]struct{}, options []string) error

func validateBaseImagePlatform(name string, expected, actual ocispecs.Platform, location []parser.Range, lint *linter.Linter) {
if expected.OS != actual.OS || expected.Architecture != actual.Architecture {
expectedStr := platforms.Format(platforms.Normalize(expected))
actualStr := platforms.Format(platforms.Normalize(actual))
expectedStr := platforms.FormatAll(platforms.Normalize(expected))
actualStr := platforms.FormatAll(platforms.Normalize(actual))
msg := linter.RuleInvalidBaseImagePlatform.Format(name, expectedStr, actualStr)
lint.Run(&linter.RuleInvalidBaseImagePlatform, location, msg)
}
Expand Down
2 changes: 2 additions & 0 deletions frontend/dockerfile/dockerfile2llb/platform.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,12 @@ func defaultArgs(po *platformOpt, overrides map[string]string, target string) *l
s := [...][2]string{
{"BUILDPLATFORM", platforms.Format(bp)},
{"BUILDOS", bp.OS},
{"BUILDOSVERSION", bp.OSVersion},
{"BUILDARCH", bp.Architecture},
{"BUILDVARIANT", bp.Variant},
{"TARGETPLATFORM", platforms.Format(tp)},
{"TARGETOS", tp.OS},
{"TARGETOSVERSION", tp.OSVersion},
{"TARGETARCH", tp.Architecture},
{"TARGETVARIANT", tp.Variant},
{"TARGETSTAGE", target},
Expand Down
145 changes: 140 additions & 5 deletions frontend/dockerfile/dockerfile_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@
testTargetStageNameArg,
testStepNames,
testPowershellInDefaultPathOnWindows,
testPlatformWithOSVersion,
)

// Tests that depend on the `security.*` entitlements
Expand Down Expand Up @@ -5000,7 +5001,7 @@
require.NoError(t, err)

dockerfile = []byte(fmt.Sprintf(`
FROM %s
FROM %s
`, target))

dir = integration.Tmpdir(
Expand Down Expand Up @@ -5218,9 +5219,9 @@
dockerfile := []byte(`
FROM alpine AS otherstage
RUN echo -n "hello" > /testfile

FROM base AS inputstage

FROM scratch
COPY --from=inputstage /out/foo /bar
`)
Expand Down Expand Up @@ -6811,7 +6812,7 @@
RUN echo $HOSTNAME | grep foo
RUN echo $(hostname) | grep foo
`,
`
`
FROM nanoserver
RUN reg query "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters" /v Hostname | findstr "foo"
`,
Expand Down Expand Up @@ -7463,7 +7464,7 @@
defer c.Close()

dockerfile := []byte(fmt.Sprintf(
`
`
FROM %s AS build
COPY <<EOF /out
hello world!
Expand Down Expand Up @@ -9327,6 +9328,140 @@
}
}

func testPlatformWithOSVersion(t *testing.T, sb integration.Sandbox) {
ctx := sb.Context()

c, err := client.New(ctx, sb.Address())
require.NoError(t, err)
defer c.Close()

f := getFrontend(t, sb)
p1 := ocispecs.Platform{
OS: "foo",
OSVersion: "1.2.3",
Architecture: "bar",
}
p2 := ocispecs.Platform{
OS: "foo",
OSVersion: "1.1.0",
Architecture: "bar",
}

p1Str := platforms.FormatAll(p1)
p2Str := platforms.FormatAll(p2)

registry, err := sb.NewRegistry()
if errors.Is(err, integration.ErrRequirements) {
t.Skip(err.Error())
}
require.NoError(t, err)
target := registry + "/buildkit/testplatformwithosversion:latest"

dockerfile := []byte(`
ARG TARGETOS TARGETOSVERSION TARGETARCH
FROM --platform=${TARGETOS}(${TARGETOSVERSION})/${TARGETARCH} ` + target + ` AS reg

FROM scratch AS base
ARG TARGETOSVERSION
COPY <<EOF /osversion
${TARGETOSVERSION}
EOF
`)

destDir := t.TempDir()
dir := integration.Tmpdir(
t,
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)

// build the base target as a multi-platform image and push to the registry
_, err = f.Solve(sb.Context(), c, client.SolveOpt{
FrontendAttrs: map[string]string{
"platform": p1Str + "," + p2Str,
"target": "base",
},
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
{
Type: client.ExporterImage,
Attrs: map[string]string{
"name": target,
"push": "true",
},
},
},

LocalMounts: map[string]fsutil.FS{
dockerui.DefaultLocalNameDockerfile: dir,
dockerui.DefaultLocalNameContext: dir,
},
}, nil)

require.NoError(t, err)

Check failure on line 9403 in frontend/dockerfile/dockerfile_test.go

View workflow job for this annotation

GitHub Actions / test-windows-amd64 (containerd, ./frontend/dockerfile)

Failed: frontend/dockerfile/TestIntegration/TestPlatformWithOSVersion/worker=containerd/frontend=builtin

=== RUN TestIntegration/TestPlatformWithOSVersion/worker=containerd/frontend=builtin dockerfile_test.go:9403: Error Trace: D:/a/buildkit/buildkit/frontend/dockerfile/dockerfile_test.go:9403 D:/a/buildkit/buildkit/util/testutil/integration/run.go:98 D:/a/buildkit/buildkit/util/testutil/integration/run.go:212 Error: Received unexpected error: Lower mount invalid: number of mounts should always be 1 for Windows layers: invalid argument github.com/moby/buildkit/util/stack.Enable D:/a/buildkit/buildkit/util/stack/stack.go:82 github.com/moby/buildkit/util/grpcerrors.FromGRPC D:/a/buildkit/buildkit/util/grpcerrors/grpcerrors.go:204 github.com/moby/buildkit/util/grpcerrors.UnaryClientInterceptor D:/a/buildkit/buildkit/util/grpcerrors/intercept.go:41 google.golang.org/grpc.(*ClientConn).Invoke D:/a/buildkit/buildkit/vendor/google.golang.org/grpc/call.go:35 github.com/moby/buildkit/api/services/control.(*controlClient).Solve D:/a/buildkit/buildkit/api/services/control/control_grpc.pb.go:88 github.com/moby/buildkit/client.(*Client).solve.func2 D:/a/buildkit/buildkit/client/solve.go:269 golang.org/x/sync/errgroup.(*Group).Go.func1 D:/a/buildkit/buildkit/vendor/golang.org/x/sync/errgroup/errgroup.go:78 runtime.goexit C:/hostedtoolcache/windows/go/1.23.4/x64/src/runtime/asm_amd64.s:1700 failed to solve github.com/moby/buildkit/client.(*Client).solve.func2 D:/a/buildkit/buildkit/client/solve.go:285 golang.org/x/sync/errgroup.(*Group).Go.func1 D:/a/buildkit/buildkit/vendor/golang.org/x/sync/errgroup/errgroup.go:78 runtime.goexit C:/hostedtoolcache/windows/go/1.23.4/x64/src/runtime/asm_amd64.s:1700 Test: TestIntegration/TestPlatformWithOSVersion/worker=containerd/frontend=builtin sandbox.go:187: stdout: D:\a\buildkit\buildkit\bin\containerd.exe --config C:\Users\RUNNER~1\AppData\Local\Temp\bktest_containerd1547712254\config.toml sandbox.go:187: stderr: D:\a\buildkit\buildkit\bin\containerd.exe --config C:\Users\RUNNER~1\AppData\Local\Temp\bktest_containerd1547712254\config.toml sandbox.go:190: > StartCmd 2024-12-28 00:05:16.8798552 +0000 UTC m=+1988.468026501 D:\a\buildkit\buildkit\bin\containerd.exe --config C:\Users\RUNNER~1\AppData\Local\Temp\bktest_containerd1547712254\config.toml sandbox.go:190: time="2024-12-28T00:05:16.899998500Z" level=info msg="starting containerd" revision= version=2.0.0+unknown sandbox.go:190: time="2024-12-28T00:05:16.899998500Z" level=debug msg="Stackdump - waiting signal at Global\\stackdump-7604" sandbox.go:190: time="2024-12-28T00:05:16.936362000Z" level=info msg="loading plugin" id=io.containerd.internal.v1.opt type=io.containerd.internal.v1 sandbox.go:190: time="2024-12-28T00:05:16.936362000Z" level=info msg="loading plugin" id=io.containerd.content.v1.content type=io.containerd.content.v1 sandbox.go:190: time="2024-12-28T00:05:16.936362000Z" level=info msg="loading plugin" id=io.containerd.image-verifier.v1.bindir type=io.containerd.image-verifier.v1 sandbox.go:190: time="2024-12-28T00:05:16.936362000Z" level=info msg="loading plugin" id=io.containerd.warning.v1.deprecations type=io.containerd.warning.v1 sandbox.go:190: time="2024-12-28T00:05:16.936362000Z" level=info msg="loading plugin" id=io.containerd.event.v1.exchange type=io.containerd.event.v1 sandbox.go:190: time="2024-12-28T00:05:16.936362000Z" level=info msg="loading plugin" id=io.containerd.snapshotter.v1.windows-lcow type=io.containerd.snapshotter.v1 sandbox.go:190: tim

dt, err := os.ReadFile(filepath.Join(destDir, strings.Replace(p1Str, "/", "_", 1), "osversion"))
require.NoError(t, err)
require.Equal(t, p1.OSVersion+"\n", string(dt))

dt, err = os.ReadFile(filepath.Join(destDir, strings.Replace(p2Str, "/", "_", 1), "osversion"))
require.NoError(t, err)
require.Equal(t, p2.OSVersion+"\n", string(dt))

// Now build the "reg" target, which should pull the base image from the registry
// This should select the image with the requested os version.
destDir = t.TempDir()
_, err = f.Solve(sb.Context(), c, client.SolveOpt{
FrontendAttrs: map[string]string{
"platform": p1Str,
"target": "reg",
},
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},

LocalMounts: map[string]fsutil.FS{
dockerui.DefaultLocalNameDockerfile: dir,
dockerui.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)

dt, err = os.ReadFile(filepath.Join(destDir, "osversion"))
require.NoError(t, err)
require.Equal(t, p1.OSVersion+"\n", string(dt))

// And again with the other os version
destDir = t.TempDir()
_, err = f.Solve(sb.Context(), c, client.SolveOpt{
FrontendAttrs: map[string]string{
"platform": p2Str,
"target": "reg",
},
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: destDir,
},
},

LocalMounts: map[string]fsutil.FS{
dockerui.DefaultLocalNameDockerfile: dir,
dockerui.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)

dt, err = os.ReadFile(filepath.Join(destDir, "osversion"))
require.NoError(t, err)
require.Equal(t, p2.OSVersion+"\n", string(dt))
}

func runShell(dir string, cmds ...string) error {
for _, args := range cmds {
var cmd *exec.Cmd
Expand Down
2 changes: 1 addition & 1 deletion frontend/dockerui/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func (bc *Client) Build(ctx context.Context, fn BuildFunc) (*ResultBuilder, erro
}

p = platforms.Normalize(p)
k := platforms.Format(p)
k := platforms.FormatAll(p)

if bc.MultiPlatformRequested {
res.AddRef(k, ref)
Expand Down
2 changes: 1 addition & 1 deletion frontend/dockerui/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ func (bc *Client) NamedContext(ctx context.Context, name string, opt ContextOpt)
if opt.Platform != nil {
pp = *opt.Platform
}
pname := name + "::" + platforms.Format(platforms.Normalize(pp))
pname := name + "::" + platforms.FormatAll(platforms.Normalize(pp))
st, img, err := bc.namedContext(ctx, name, pname, opt)
if err != nil || st != nil {
return st, img, err
Expand Down
4 changes: 2 additions & 2 deletions solver/llbsolver/bridge.go
Original file line number Diff line number Diff line change
Expand Up @@ -354,9 +354,9 @@ func (b *llbBridge) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp,
}
id := op.Identifier
if opt.Platform != nil {
id += platforms.Format(*opt.Platform)
id += platforms.FormatAll(*opt.Platform)
} else {
id += platforms.Format(platforms.DefaultSpec())
id += platforms.FormatAll(platforms.DefaultSpec())
}
pol, err := loadSourcePolicy(b.builder)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion source/containerimage/source.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ func (is *Source) ResolveImageConfig(ctx context.Context, ref string, opt source
err error
)
if platform := opt.Platform; platform != nil {
key += platforms.Format(*platform)
key += platforms.FormatAll(*platform)
}

switch is.ResolverType {
Expand Down
Loading
Loading