wip: kaniko

Signed-off-by: Anthony Emengo <[email protected]>

Makefile

@@ -42,9 +42,9 @@ all: test build package
 
 build: build-linux-amd64 build-linux-arm64 build-windows-amd64
 
-build-linux-amd64: build-linux-amd64-lifecycle build-linux-amd64-symlinks build-linux-amd64-launcher
-build-linux-arm64: build-linux-arm64-lifecycle build-linux-arm64-symlinks build-linux-arm64-launcher
-build-windows-amd64: build-windows-amd64-lifecycle build-windows-amd64-symlinks build-windows-amd64-launcher
+build-linux-amd64: build-linux-amd64-lifecycle build-linux-amd64-symlinks build-linux-amd64-launcher build-linux-amd64-extender
+build-linux-arm64: build-linux-arm64-lifecycle build-linux-arm64-symlinks build-linux-arm64-launcher build-linux-arm64-extender
+build-windows-amd64: build-windows-amd64-lifecycle build-windows-amd64-symlinks build-windows-amd64-launcher build-windows-amd64-extender
 
 build-image-linux-amd64: build-linux-amd64 package-linux-amd64
 build-image-linux-amd64: ARCHIVE_PATH=$(BUILD_DIR)/lifecycle-v$(LIFECYCLE_VERSION)+linux.x86-64.tgz
@@ -107,6 +107,28 @@ $(BUILD_DIR)/linux-arm64/lifecycle/launcher:
 	$(GOENV) $(GOBUILD) -o $(OUT_DIR)/launcher -a ./cmd/launcher
 	test $$(du -m $(OUT_DIR)/launcher|cut -f 1) -le 3
 
+build-linux-amd64-extender: $(BUILD_DIR)/linux-amd64/lifecycle/extender
+
+$(BUILD_DIR)/linux-amd64/lifecycle/extender: export GOOS:=linux
+$(BUILD_DIR)/linux-amd64/lifecycle/extender: export GOARCH:=amd64
+$(BUILD_DIR)/linux-amd64/lifecycle/extender: OUT_DIR?=$(BUILD_DIR)/$(GOOS)-$(GOARCH)/lifecycle
+$(BUILD_DIR)/linux-amd64/lifecycle/extender: $(GOFILES)
+$(BUILD_DIR)/linux-amd64/lifecycle/extender:
+	@echo "> Building lifecycle/extender for $(GOOS)/$(GOARCH)..."
+	mkdir -p $(OUT_DIR)
+	cd ./extender && $(GOENV) $(GOBUILD) -o $(OUT_DIR)/extender -a .
+
+build-linux-arm64-extender: $(BUILD_DIR)/linux-arm64/lifecycle/extender
+
+$(BUILD_DIR)/linux-arm64/lifecycle/extender: export GOOS:=linux
+$(BUILD_DIR)/linux-arm64/lifecycle/extender: export GOARCH:=arm64
+$(BUILD_DIR)/linux-arm64/lifecycle/extender: OUT_DIR?=$(BUILD_DIR)/$(GOOS)-$(GOARCH)/lifecycle
+$(BUILD_DIR)/linux-arm64/lifecycle/extender: $(GOFILES)
+$(BUILD_DIR)/linux-arm64/lifecycle/extender:
+	@echo "> Building lifecycle/extender for $(GOOS)/$(GOARCH)..."
+	mkdir -p $(OUT_DIR)
+	cd ./extender && $(GOENV) $(GOBUILD) -o $(OUT_DIR)/extender -a .
+
 build-linux-amd64-symlinks: export GOOS:=linux
 build-linux-amd64-symlinks: export GOARCH:=amd64
 build-linux-amd64-symlinks: OUT_DIR?=$(BUILD_DIR)/$(GOOS)-$(GOARCH)/lifecycle
@@ -153,6 +175,18 @@ $(BUILD_DIR)/windows-amd64/lifecycle/launcher.exe:
 	@echo "> Building lifecycle/launcher for $(GOOS)/$(GOARCH)..."
 	$(GOBUILD) -o $(OUT_DIR)$/launcher.exe -a .$/cmd$/launcher
 
+# ANTHONY: The following (build extender for windows) does not work
+#          Someone should get back to this
+build-windows-amd64-extender: $(BUILD_DIR)/windows-amd64/lifecycle/extender.exe
+
+$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: export GOOS:=windows
+$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: export GOARCH:=amd64
+$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: OUT_DIR?=$(BUILD_DIR)$/$(GOOS)-$(GOARCH)$/lifecycle
+$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: $(GOFILES)
+$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe:
+	@echo "> Building lifecycle/extender for $(GOOS)/$(GOARCH)..."
+	cd .$/extender && $(GOBUILD) -o $(OUT_DIR)$/extender.exe -a .
+
 build-windows-amd64-symlinks: export GOOS:=windows
 build-windows-amd64-symlinks: export GOARCH:=amd64
 build-windows-amd64-symlinks: OUT_DIR?=$(BUILD_DIR)$/$(GOOS)-$(GOARCH)$/lifecycle

cmd/lifecycle/creator.go

@@ -239,6 +239,14 @@ func (c *createCmd) Exec() error {
 		}
 	}
 
+	cmd.DefaultLogger.Phase("EXTENDING")
+	err = extendArgs{
+		platform: c.platform,
+	}.extend()
+	if err != nil {
+		return err
+	}
+
 	// send pings to docker daemon while BUILDING to prevent connection closure
 	stopPinging := startPinging(c.docker)
 	cmd.DefaultLogger.Phase("BUILDING")

cmd/lifecycle/extender.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"github.com/buildpacks/lifecycle"
+	"github.com/buildpacks/lifecycle/cmd"
+)
+
+type extendCmd struct {
+	extendArgs
+}
+
+type extendArgs struct {
+	platform Platform
+	mode     string
+}
+
+func (e *extendCmd) DefineFlags() {
+	// no-op
+}
+
+func (e *extendCmd) Args(nargs int, args []string) error {
+	if len(args) == 0 {
+		return nil
+	}
+
+	e.extendArgs.mode = args[0]
+	return nil
+}
+
+func (e *extendCmd) Privileges() error {
+	return nil
+}
+
+func (e *extendCmd) Exec() error {
+	return e.extend()
+}
+
+func (e extendArgs) extend() error {
+	extender := &lifecycle.Extender{
+		Logger: cmd.DefaultLogger,
+		Mode:   e.mode,
+	}
+
+	return extender.Extend()
+}

cmd/lifecycle/main.go

@@ -35,6 +35,8 @@ func main() {
 		cmd.Run(&analyzeCmd{analyzeArgs: analyzeArgs{platform: p}}, false)
 	case "restorer":
 		cmd.Run(&restoreCmd{restoreArgs: restoreArgs{platform: p}}, false)
+	case "extender":
+		cmd.Run(&extendCmd{extendArgs: extendArgs{platform: p}}, false)
 	case "builder":
 		cmd.Run(&buildCmd{buildArgs: buildArgs{platform: p}}, false)
 	case "exporter":
@@ -63,6 +65,8 @@ func subcommand(platform Platform) {
 		cmd.Run(&analyzeCmd{analyzeArgs: analyzeArgs{platform: platform}}, true)
 	case "restore":
 		cmd.Run(&restoreCmd{restoreArgs: restoreArgs{platform: platform}}, true)
+	case "extend":
+		cmd.Run(&extendCmd{extendArgs: extendArgs{platform: platform}}, true)
 	case "build":
 		cmd.Run(&buildCmd{buildArgs: buildArgs{platform: platform}}, true)
 	case "export":

exporter.go

@@ -3,15 +3,14 @@ package lifecycle
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/BurntSushi/toml"
+	"github.com/buildpacks/imgutil"
+	"github.com/pkg/errors"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/BurntSushi/toml"
-	"github.com/buildpacks/imgutil"
-	"github.com/pkg/errors"
-
 	"github.com/buildpacks/lifecycle/api"
 	"github.com/buildpacks/lifecycle/buildpack"
 	"github.com/buildpacks/lifecycle/cmd"
@@ -91,6 +90,11 @@ func (e *Exporter) Export(opts ExportOptions) (platform.ExportReport, error) {
 		return platform.ExportReport{}, errors.Wrap(err, "read build metadata")
 	}
 
+	// extender layers
+	if err := e.addExtenderLayers(opts, &meta); err != nil {
+		return platform.ExportReport{}, err
+	}
+
 	// buildpack-provided layers
 	if err := e.addBuildpackLayers(opts, &meta); err != nil {
 		return platform.ExportReport{}, err
@@ -504,3 +508,46 @@ func (e *Exporter) addSBOMLaunchLayer(opts ExportOptions, meta *platform.LayersM
 
 	return nil
 }
+
+func (e *Exporter) addExtenderLayers(opts ExportOptions, meta *platform.LayersMetadata) error {
+	manifestPath := "/layers/kaniko/manifest.json"
+	var manifest []struct {
+		Layers []string `json:"Layers"`
+	}
+
+	f, err := os.Open(manifestPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+
+		return err
+	}
+
+	defer f.Close()
+
+	err = json.NewDecoder(f).Decode(&manifest)
+	if err != nil {
+		return err
+	}
+
+	if len(manifest) == 0 || len(manifest[0].Layers) == 0 {
+		return nil
+	}
+
+	// Iterate layers except first one (the base layer)
+	for _, layerName := range manifest[0].Layers[1:] {
+		layer := layers.Layer{
+			ID:      strings.TrimSuffix(layerName, ".tgz"),
+			TarPath: "/layers/kaniko/" + layerName,
+			Digest:  strings.TrimSuffix(layerName, ".tgz"),
+		}
+
+		_, err := e.addOrReuseLayer(opts.WorkingImage, layer, "")
+		if err != nil {
+			return errors.Wrapf(err, "exporting kaniko layer '%s'", layer.ID)
+		}
+	}
+
+	return nil
+}

extender.go

@@ -0,0 +1,23 @@
+package lifecycle
+
+import (
+	"os"
+	"os/exec"
+)
+
+type Extender struct {
+	Logger Logger
+	Mode   string
+}
+
+func (e *Extender) Extend() error {
+	mode := "kaniko"
+	if e.Mode != ""{
+		mode = e.Mode
+	}
+
+	cmd := exec.Command("/cnb/lifecycle/extender", mode)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}

extender/go.mod

@@ -0,0 +1,15 @@
+module github.com/buildpacks/lifecycle/extender
+
+go 1.16
+
+require (
+	github.com/redhat-buildpacks/poc/kaniko v0.0.0-00010101000000-000000000000
+)
+
+replace (
+	github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0+incompatible
+	github.com/containerd/containerd v1.4.0-0.20191014053712-acdcf13d5eaf => github.com/containerd/containerd v0.0.0-20191014053712-acdcf13d5eaf
+	github.com/docker/docker v1.14.0-0.20190319215453-e7b5f7dbe98c => github.com/docker/docker v0.0.0-20190319215453-e7b5f7dbe98c
+	github.com/redhat-buildpacks/poc/kaniko => ../../poc/kaniko/code
+	github.com/tonistiigi/fsutil v0.0.0-20190819224149-3d2716dd0a4d => github.com/tonistiigi/fsutil v0.0.0-20191018213012-0f039a052ca1
+)