wip: Anthony + Natalie: merging in charles POC kaniko

Signed-off-by: Anthony Emengo <[email protected]>

Makefile

@@ -42,9 +42,9 @@ all: test build package
 
 build: build-linux-amd64 build-linux-arm64 build-windows-amd64
 
-build-linux-amd64: build-linux-amd64-lifecycle build-linux-amd64-symlinks build-linux-amd64-launcher
-build-linux-arm64: build-linux-arm64-lifecycle build-linux-arm64-symlinks build-linux-arm64-launcher
-build-windows-amd64: build-windows-amd64-lifecycle build-windows-amd64-symlinks build-windows-amd64-launcher
+build-linux-amd64: build-linux-amd64-lifecycle build-linux-amd64-symlinks build-linux-amd64-launcher build-linux-amd64-extender
+build-linux-arm64: build-linux-arm64-lifecycle build-linux-arm64-symlinks build-linux-arm64-launcher build-linux-arm64-extender
+build-windows-amd64: build-windows-amd64-lifecycle build-windows-amd64-symlinks build-windows-amd64-launcher build-windows-amd64-extender
 
 build-image-linux-amd64: build-linux-amd64 package-linux-amd64
 build-image-linux-amd64: ARCHIVE_PATH=$(BUILD_DIR)/lifecycle-v$(LIFECYCLE_VERSION)+linux.x86-64.tgz
@@ -107,6 +107,28 @@ $(BUILD_DIR)/linux-arm64/lifecycle/launcher:
 	$(GOENV) $(GOBUILD) -o $(OUT_DIR)/launcher -a ./cmd/launcher
 	test $$(du -m $(OUT_DIR)/launcher|cut -f 1) -le 3
 
+build-linux-amd64-extender: $(BUILD_DIR)/linux-amd64/lifecycle/extender
+
+$(BUILD_DIR)/linux-amd64/lifecycle/extender: export GOOS:=linux
+$(BUILD_DIR)/linux-amd64/lifecycle/extender: export GOARCH:=amd64
+$(BUILD_DIR)/linux-amd64/lifecycle/extender: OUT_DIR?=$(BUILD_DIR)/$(GOOS)-$(GOARCH)/lifecycle
+$(BUILD_DIR)/linux-amd64/lifecycle/extender: $(GOFILES)
+$(BUILD_DIR)/linux-amd64/lifecycle/extender:
+	@echo "> Building lifecycle/extender for $(GOOS)/$(GOARCH)..."
+	mkdir -p $(OUT_DIR)
+	cd ./extender && $(GOENV) $(GOBUILD) -o $(OUT_DIR)/extender -a .
+
+build-linux-arm64-extender: $(BUILD_DIR)/linux-arm64/lifecycle/extender
+
+$(BUILD_DIR)/linux-arm64/lifecycle/extender: export GOOS:=linux
+$(BUILD_DIR)/linux-arm64/lifecycle/extender: export GOARCH:=arm64
+$(BUILD_DIR)/linux-arm64/lifecycle/extender: OUT_DIR?=$(BUILD_DIR)/$(GOOS)-$(GOARCH)/lifecycle
+$(BUILD_DIR)/linux-arm64/lifecycle/extender: $(GOFILES)
+$(BUILD_DIR)/linux-arm64/lifecycle/extender:
+	@echo "> Building lifecycle/extender for $(GOOS)/$(GOARCH)..."
+	mkdir -p $(OUT_DIR)
+	cd ./extender && $(GOENV) $(GOBUILD) -o $(OUT_DIR)/extender -a .
+
 build-linux-amd64-symlinks: export GOOS:=linux
 build-linux-amd64-symlinks: export GOARCH:=amd64
 build-linux-amd64-symlinks: OUT_DIR?=$(BUILD_DIR)/$(GOOS)-$(GOARCH)/lifecycle
@@ -153,6 +175,18 @@ $(BUILD_DIR)/windows-amd64/lifecycle/launcher.exe:
 	@echo "> Building lifecycle/launcher for $(GOOS)/$(GOARCH)..."
 	$(GOBUILD) -o $(OUT_DIR)$/launcher.exe -a .$/cmd$/launcher
 
+# ANTHONY: The following (build extender for windows) does not work
+#          Someone should get back to this
+build-windows-amd64-extender: $(BUILD_DIR)/windows-amd64/lifecycle/extender.exe
+
+$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: export GOOS:=windows
+$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: export GOARCH:=amd64
+$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: OUT_DIR?=$(BUILD_DIR)$/$(GOOS)-$(GOARCH)$/lifecycle
+$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe: $(GOFILES)
+$(BUILD_DIR)/windows-amd64/lifecycle/extender.exe:
+	@echo "> Building lifecycle/extender for $(GOOS)/$(GOARCH)..."
+	cd .$/extender && $(GOBUILD) -o $(OUT_DIR)$/extender.exe -a .
+
 build-windows-amd64-symlinks: export GOOS:=windows
 build-windows-amd64-symlinks: export GOARCH:=amd64
 build-windows-amd64-symlinks: OUT_DIR?=$(BUILD_DIR)$/$(GOOS)-$(GOARCH)$/lifecycle

builder.go

@@ -2,20 +2,18 @@ package lifecycle
 
 import (
 	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"sort"
-
-	"github.com/pkg/errors"
-
 	"github.com/buildpacks/lifecycle/api"
 	"github.com/buildpacks/lifecycle/buildpack"
 	"github.com/buildpacks/lifecycle/env"
 	io2 "github.com/buildpacks/lifecycle/internal/io"
 	"github.com/buildpacks/lifecycle/launch"
 	"github.com/buildpacks/lifecycle/layers"
 	"github.com/buildpacks/lifecycle/platform"
+	"github.com/pkg/errors"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
 )
 
 type BuildEnv interface {
@@ -123,6 +121,7 @@ func (b *Builder) Build() (*platform.BuildMetadata, error) {
 	procList := processMap.list()
 
 	b.Logger.Debug("Finished build")
+
 	return &platform.BuildMetadata{
 		BOM:                         bom,
 		BuildpackDefaultProcessType: processMap.defaultType,

cmd/lifecycle/builder.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"errors"
-
 	"github.com/BurntSushi/toml"
 
 	"github.com/buildpacks/lifecycle"

cmd/lifecycle/creator.go

@@ -238,6 +238,14 @@ func (c *createCmd) Exec() error {
 		}
 	}
 
+	cmd.DefaultLogger.Phase("EXTENDING")
+	err = extendArgs{
+		platform: c.platform,
+	}.extend()
+	if err != nil {
+		return err
+	}
+
 	// send pings to docker daemon while BUILDING to prevent connection closure
 	stopPinging := startPinging(c.docker)
 	cmd.DefaultLogger.Phase("BUILDING")

cmd/lifecycle/extender.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"github.com/buildpacks/lifecycle"
+	"github.com/buildpacks/lifecycle/cmd"
+)
+
+type extendCmd struct {
+	extendArgs
+}
+
+type extendArgs struct {
+	platform Platform
+	mode     string
+}
+
+func (e *extendCmd) DefineFlags() {
+	// no-op
+}
+
+func (e *extendCmd) Args(nargs int, args []string) error {
+	if len(args) == 0 {
+		return nil
+	}
+
+	e.extendArgs.mode = args[0]
+	return nil
+}
+
+func (e *extendCmd) Privileges() error {
+	return nil
+}
+
+func (e *extendCmd) Exec() error {
+	return e.extend()
+}
+
+func (e extendArgs) extend() error {
+	extender := &lifecycle.Extender{
+		Logger: cmd.DefaultLogger,
+		Mode:   e.mode,
+	}
+
+	return extender.Extend()
+}

cmd/lifecycle/main.go

@@ -35,6 +35,8 @@ func main() {
 		cmd.Run(&analyzeCmd{analyzeArgs: analyzeArgs{platform: p}}, false)
 	case "restorer":
 		cmd.Run(&restoreCmd{restoreArgs: restoreArgs{platform: p}}, false)
+	case "extender":
+		cmd.Run(&extendCmd{extendArgs: extendArgs{platform: p}}, false)
 	case "builder":
 		cmd.Run(&buildCmd{buildArgs: buildArgs{platform: p}}, false)
 	case "exporter":
@@ -63,6 +65,8 @@ func subcommand(platform Platform) {
 		cmd.Run(&analyzeCmd{analyzeArgs: analyzeArgs{platform: platform}}, true)
 	case "restore":
 		cmd.Run(&restoreCmd{restoreArgs: restoreArgs{platform: platform}}, true)
+	case "extend":
+		cmd.Run(&extendCmd{extendArgs: extendArgs{platform: platform}}, true)
 	case "build":
 		cmd.Run(&buildCmd{buildArgs: buildArgs{platform: platform}}, true)
 	case "export":

exporter.go

@@ -3,15 +3,14 @@ package lifecycle
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/pkg/errors"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/BurntSushi/toml"
 	"github.com/buildpacks/imgutil"
-	"github.com/pkg/errors"
-
 	"github.com/buildpacks/lifecycle/api"
 	"github.com/buildpacks/lifecycle/buildpack"
 	"github.com/buildpacks/lifecycle/cmd"
@@ -91,6 +90,11 @@ func (e *Exporter) Export(opts ExportOptions) (platform.ExportReport, error) {
 		return platform.ExportReport{}, errors.Wrap(err, "read build metadata")
 	}
 
+	// extender layers
+	if err := e.addExtenderLayers(opts, &meta); err != nil {
+		return platform.ExportReport{}, err
+	}
+
 	// buildpack-provided layers
 	if err := e.addBuildpackLayers(opts, &meta); err != nil {
 		return platform.ExportReport{}, err
@@ -504,3 +508,50 @@ func (e *Exporter) addSBOMLaunchLayer(opts ExportOptions, meta *platform.LayersM
 
 	return nil
 }
+
+func (e *Exporter) addExtenderLayers(opts ExportOptions, meta *platform.LayersMetadata) error {
+	manifestPath := "/layers/kaniko/manifest.json"
+	var manifest []struct {
+		Layers []string `json:"Layers"`
+	}
+
+	f, err := os.Open(manifestPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+
+		return err
+	}
+
+	defer f.Close()
+
+	err = json.NewDecoder(f).Decode(&manifest)
+	if err != nil {
+		return err
+	}
+
+	if len(manifest) == 0 || len(manifest[0].Layers) == 0 {
+		return nil
+	}
+
+	// Iterate layers except first one (the base layer)
+	for _, layerName := range manifest[0].Layers[1:] {
+		dir := layerDir{
+			path:       "/layers/kaniko/" + strings.TrimSuffix(layerName, ".tar.gz"),
+			identifier: strings.TrimSuffix(layerName, ".tar.gz"),
+		}
+
+		layer, err := e.LayerFactory.DirLayer(dir.Identifier(), dir.Path())
+		if err != nil {
+			return errors.Wrapf(err, "creating layer")
+		}
+
+		_, err = e.addOrReuseLayer(opts.WorkingImage, layer, "")
+		if err != nil {
+			return errors.Wrapf(err, "exporting layer '%s'", layer.ID)
+		}
+	}
+
+	return nil
+}

extender.go

@@ -0,0 +1,23 @@
+package lifecycle
+
+import (
+	"os"
+	"os/exec"
+)
+
+type Extender struct {
+	Logger Logger
+	Mode   string
+}
+
+func (e *Extender) Extend() error {
+	mode := "kaniko"
+	if e.Mode != ""{
+		mode = e.Mode
+	}
+
+	cmd := exec.Command("/cnb/lifecycle/extender", mode)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}

extender/go.mod

@@ -0,0 +1,17 @@
+module github.com/buildpacks/lifecycle/extender
+
+go 1.16
+
+require (
+	github.com/BurntSushi/toml v0.3.1
+	github.com/GoogleContainerTools/kaniko v1.7.0
+	github.com/redhat-buildpacks/poc/kaniko v0.0.0-00010101000000-000000000000
+)
+
+replace (
+	github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.2.0+incompatible
+	github.com/containerd/containerd v1.4.0-0.20191014053712-acdcf13d5eaf => github.com/containerd/containerd v0.0.0-20191014053712-acdcf13d5eaf
+	github.com/docker/docker v1.14.0-0.20190319215453-e7b5f7dbe98c => github.com/docker/docker v0.0.0-20190319215453-e7b5f7dbe98c
+	github.com/redhat-buildpacks/poc/kaniko => ../../poc/kaniko/code
+	github.com/tonistiigi/fsutil v0.0.0-20190819224149-3d2716dd0a4d => github.com/tonistiigi/fsutil v0.0.0-20191018213012-0f039a052ca1
+)