From ba7462f01b7779aec79861e3d1736c6e30b5277d Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 20 Mar 2024 21:06:27 +0000 Subject: [PATCH] WIP: docker bridge --- go.mod | 6 ++-- go.sum | 4 +++ pkg/buildkit/buildkit_test.go | 4 +-- pkg/buildkit/connhelpers/buildx.go | 29 ++++++++++++++++++- pkg/buildkit/connhelpers/docker.go | 2 ++ pkg/buildkit/drivers.go | 35 ++++++++++++++++------- pkg/patch/patch.go | 45 +++++++++++++++--------------- pkg/pkgmgr/dpkg.go | 2 -- pkg/pkgmgr/rpm.go | 2 -- 9 files changed, 87 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index ce0862c1..a3723f4e 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,11 @@ toolchain go1.21.5 require ( github.com/aquasecurity/trivy v0.45.1 + github.com/containerd/containerd v1.7.13 + github.com/containerd/errdefs v0.1.0 github.com/cpuguy83/dockercfg v0.3.1 github.com/cpuguy83/go-docker v0.3.0 + github.com/cpuguy83/tar2go v0.3.1 github.com/distribution/reference v0.5.0 github.com/docker/buildx v0.13.1 github.com/docker/cli v26.0.0-rc1+incompatible @@ -20,6 +23,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0 github.com/openvex/go-vex v0.2.5 + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 @@ -55,7 +59,6 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/console v1.0.4 // indirect - github.com/containerd/containerd v1.7.13 // indirect github.com/containerd/continuity v0.4.3 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/ttrpc v1.2.2 // indirect @@ -109,7 +112,6 @@ require ( github.com/package-url/packageurl-go v0.1.2-0.20230812223828-f8bb31c1f10b // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect diff --git a/go.sum b/go.sum index cdff377e..d3e8a7ee 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/containerd/containerd v1.7.13 h1:wPYKIeGMN8vaggSKuV1X0wZulpMz4CrgEsZd github.com/containerd/containerd v1.7.13/go.mod h1:zT3up6yTRfEUa6+GsITYIJNgSVL9NQ4x4h1RPzk0Wu4= github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= +github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= @@ -107,6 +109,8 @@ github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHf github.com/cpuguy83/go-docker v0.3.0 h1:O88rocdycYvY+pUYYp0i1rRDANXHurNir3VE0F/PH3g= github.com/cpuguy83/go-docker v0.3.0/go.mod h1:R2HgB/m54W+2dhYc70Xm78yS6o775SfN09bGIPSfQZQ= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/tar2go v0.3.1 h1:DMWlaIyoh9FBWR4hyfZSOEDA7z8rmCiGF1IJIzlTlR8= +github.com/cpuguy83/tar2go v0.3.1/go.mod h1:2Ys2/Hu+iPHQRa4DjIVJ7UAaKnDhAhNACeK3A0Rr5rM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= diff --git a/pkg/buildkit/buildkit_test.go b/pkg/buildkit/buildkit_test.go index 8f9279d6..ebe6b436 100644 --- a/pkg/buildkit/buildkit_test.go +++ b/pkg/buildkit/buildkit_test.go @@ -187,7 +187,7 @@ func TestNewClient(t *testing.T) { bkOpts := Opts{ Addr: "unix://" + addr, } - client, err := NewClient(ctxT, bkOpts) + client, _, err := NewClient(ctxT, bkOpts) cancel() assert.NoError(t, err) defer client.Close() @@ -206,7 +206,7 @@ func TestNewClient(t *testing.T) { bkOpts := Opts{ Addr: "unix://" + addr, } - client, err := NewClient(ctxT, bkOpts) + client, _, err := NewClient(ctxT, bkOpts) assert.NoError(t, err) defer client.Close() diff --git a/pkg/buildkit/connhelpers/buildx.go b/pkg/buildkit/connhelpers/buildx.go index 0f0c6a99..1679aff8 100644 --- a/pkg/buildkit/connhelpers/buildx.go +++ b/pkg/buildkit/connhelpers/buildx.go @@ -12,11 +12,13 @@ import ( "os" "os/exec" "path/filepath" + "sync" "github.com/cpuguy83/dockercfg" "github.com/cpuguy83/go-docker" "github.com/cpuguy83/go-docker/container" "github.com/cpuguy83/go-docker/errdefs" + "github.com/cpuguy83/go-docker/transport" "github.com/moby/buildkit/client/connhelper" log "github.com/sirupsen/logrus" ) @@ -117,12 +119,18 @@ func buildxContextDialer(builder string) func(context.Context, string) (net.Conn } } -func containerContextDialer(ctx context.Context, host, name string) (net.Conn, error) { +func containerContextDialer(ctx context.Context, host, name string) (_ net.Conn, retErr error) { tr, err := getDockerTransport(host) if err != nil { return nil, err } + defer func() { + if retErr == nil { + setDockerBridgeTransport(tr) + } + }() + cli := docker.NewClient(docker.WithTransport(tr)) c := cli.ContainerService().NewContainer(ctx, name) @@ -156,3 +164,22 @@ func containerContextDialer(ctx context.Context, host, name string) (net.Conn, e return conn2, nil } + +// GetDockerTransport returns the configured transport for connecting to a docker daemon. +// This may be nil if no transport has been configured. +func GetDockerBridgeTransport() transport.Doer { + dockerTransportMu.Lock() + defer dockerTransportMu.Unlock() + return dockerTransport +} + +var ( + dockerTransportMu sync.Mutex + dockerTransport transport.Doer +) + +func setDockerBridgeTransport(t transport.Doer) { + dockerTransportMu.Lock() + dockerTransport = t + dockerTransportMu.Unlock() +} diff --git a/pkg/buildkit/connhelpers/docker.go b/pkg/buildkit/connhelpers/docker.go index c4a91dc6..3365092e 100644 --- a/pkg/buildkit/connhelpers/docker.go +++ b/pkg/buildkit/connhelpers/docker.go @@ -25,11 +25,13 @@ func Docker(u *url.URL) (*connhelper.ConnectionHelper, error) { if err != nil { return nil, err } + return tr.DoRaw(ctx, http.MethodPost, version.Join(ctx, "/grpc"), transport.WithUpgrade("h2c")) }, }, nil } +// GetDockerTransport returns a transport for connecting to a docker daemon. func getDockerTransport(addr string) (transport.Doer, error) { if addr == "" { addr = os.Getenv("DOCKER_HOST") diff --git a/pkg/buildkit/drivers.go b/pkg/buildkit/drivers.go index ac14d1a3..39d306a7 100644 --- a/pkg/buildkit/drivers.go +++ b/pkg/buildkit/drivers.go @@ -7,6 +7,7 @@ import ( "net" "net/url" + "github.com/cpuguy83/go-docker/transport" "github.com/moby/buildkit/client" gateway "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/solver/pb" @@ -27,16 +28,25 @@ var ( // NewClient returns a new buildkit client with the given addr. // If addr is empty it will first try to connect to docker's buildkit instance and then fallback to DefaultAddr. -func NewClient(ctx context.Context, bkOpts Opts) (*client.Client, error) { +// +// Whent he returned boolean is true, that denotes that the client is using the buildkit instance embedded in dockerd. +func NewClient(ctx context.Context, bkOpts Opts) (*client.Client, transport.Doer, error) { if bkOpts.Addr == "" { return autoClient(ctx) } opts := getCredentialOptions(bkOpts) client, err := client.New(ctx, bkOpts.Addr, opts...) if err != nil { - return nil, err + return nil, nil, err + } + + // Make sure the docker bridge transport is set (if needed) + if _, err := client.Info(ctx); err != nil { + client.Close() + return nil, nil, err } - return client, nil + + return client, connhelpers.GetDockerBridgeTransport(), nil } func getCredentialOptions(bkOpts Opts) []client.ClientOpt { @@ -76,7 +86,7 @@ func ValidateClient(ctx context.Context, c *client.Client) error { return err } -func autoClient(ctx context.Context, opts ...client.ClientOpt) (*client.Client, error) { +func autoClient(ctx context.Context, opts ...client.ClientOpt) (*client.Client, transport.Doer, error) { var retErr error newClient := func(ctx context.Context, dialer func(context.Context, string) (net.Conn, error)) (*client.Client, error) { @@ -94,11 +104,12 @@ func autoClient(ctx context.Context, opts ...client.ClientOpt) (*client.Client, log.Debug("Trying docker driver") h, err := connhelpers.Docker(&url.URL{}) if err != nil { - return nil, err + return nil, nil, err } + c, err := newClient(ctx, h.ContextDialer) if err == nil { - return c, nil + return c, nil, nil } log.WithError(err).Debug("Could not use docker driver") retErr = errors.Join(retErr, fmt.Errorf("could not use docker driver: %w", err)) @@ -106,12 +117,12 @@ func autoClient(ctx context.Context, opts ...client.ClientOpt) (*client.Client, log.Debug("Trying buildx driver") h, err = connhelpers.Buildx(&url.URL{}) if err != nil { - return nil, err + return nil, nil, err } c, err = newClient(ctx, h.ContextDialer) if err == nil { - return c, nil + return c, connhelpers.GetDockerBridgeTransport(), nil } log.WithError(err).Debug("Could not use buildx driver") retErr = errors.Join(retErr, fmt.Errorf("could not use buildx driver: %w", err)) @@ -120,10 +131,14 @@ func autoClient(ctx context.Context, opts ...client.ClientOpt) (*client.Client, c, err = client.New(ctx, DefaultAddr, opts...) if err == nil { if err := ValidateClient(ctx, c); err == nil { - return c, nil + tr, err := transport.DefaultTransport() + if err != nil { + log.WithError(err).Warn("Could not setup docker bridge transport") + } + return c, tr, nil } c.Close() } log.WithError(err).Debug("Could not use buildkitd driver") - return nil, errors.Join(retErr, fmt.Errorf("could not use buildkitd driver: %w", err)) + return nil, nil, errors.Join(retErr, fmt.Errorf("could not use buildkitd driver: %w", err)) } diff --git a/pkg/patch/patch.go b/pkg/patch/patch.go index 8eebb2dd..b19c70a9 100644 --- a/pkg/patch/patch.go +++ b/pkg/patch/patch.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "os" - "os/exec" "time" "github.com/docker/buildx/build" @@ -126,7 +125,7 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat } log.Debugf("updates to apply: %v", updates) - bkClient, err := buildkit.NewClient(ctx, bkOpts) + bkClient, dockerTr, err := buildkit.NewClient(ctx, bkOpts) if err != nil { return err } @@ -155,11 +154,32 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat return err } + gwcWrap := func(gwc gwclient.Client) gwclient.Client { + // no-op + return gwc + } + + if dockerTr != nil { + log.Info("Using docker bridge") + layout, wrap, err := fetchImageFromDocker(ctx, &solveOpt, dockerTr, image, imageName) + if err != nil { + return err + } + gwcWrap = wrap + defer func() { + if err := os.RemoveAll(layout); err != nil { + log.WithError(err).WithField("layout", layout).Warn("Failed to remove temporary image layout") + } + }() + } + buildChannel := make(chan *client.SolveStatus) eg, ctx := errgroup.WithContext(ctx) + eg.Go(func() error { _, err := bkClient.Build(ctx, solveOpt, copaProduct, func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) { // Configure buildctl/client for use by package manager + c = gwcWrap(c) config, err := buildkit.InitializeBuildkitConfig(ctx, c, imageName.String(), updates) if err != nil { ch <- err @@ -254,24 +274,3 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat return eg.Wait() } - -func dockerLoad(ctx context.Context, pipeR io.Reader) error { - cmd := exec.CommandContext(ctx, "docker", "load") - cmd.Stdin = pipeR - - stdout, err := cmd.StdoutPipe() - if err != nil { - return err - } - stderr, err := cmd.StderrPipe() - if err != nil { - return err - } - - // Pipe run errors to WarnLevel since execution continues asynchronously - // Caller should log a separate ErrorLevel on completion based on err - go utils.LogPipe(stderr, log.WarnLevel) - go utils.LogPipe(stdout, log.InfoLevel) - - return cmd.Run() -} diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index df66803e..6a0e0bca 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -150,7 +150,6 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string) er // Spin up a build tooling container to pull and unpack packages to create patch layer. toolingBase := llb.Image(toolImage, llb.Platform(dm.config.Platform), - llb.ResolveModeDefault, ) updated := toolingBase.Run( llb.Shlex("apt update"), @@ -265,7 +264,6 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers // Pull family:version -> need to create version to base image map toolingBase := llb.Image(toolImage, llb.Platform(dm.config.Platform), - llb.ResolveModeDefault, ) // Run apt update && apt download list of updates to target folder diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index 810084c7..5501807a 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -214,7 +214,6 @@ func (rm *rpmManager) probeRPMStatus(ctx context.Context, toolImage string) erro // Spin up a build tooling container to pull and unpack packages to create patch layer. toolingBase := llb.Image(toolImage, llb.Platform(rm.config.Platform), - llb.ResolveModeDefault, ) toolsInstalled := toolingBase.Run(llb.Shlex(installToolsCmd), llb.WithProxy(utils.GetProxy())).Root() @@ -367,7 +366,6 @@ func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversi // Pull family:version -> need to create version to base image map toolingBase := llb.Image(toolImage, llb.Platform(rm.config.Platform), - llb.ResolveModeDefault, ) // Install busybox. This should reuse the layer cached from probeRPMStatus.