Container Image Caching (#505)

* added image caching

* fix testing

* feedback updates

RELEASE_NOTES.md

@@ -9,6 +9,7 @@
 * Artifact sources origin and metadata are now extracted from a configuration file (`config/artifacts.yaml`)
 * Dropped `-chart` suffix from installed Helm chart names
 * Added ability to build aarch64 images on an aarch64 host machine
+* Added caching for container images
 
 ## API
 

pkg/cache/cache.go

@@ -17,12 +17,7 @@ type Cache struct {
 	cacheDir string
 }
 
-func New(rootDir string) (*Cache, error) {
-	cacheDir := filepath.Join(rootDir, "cache")
-	if err := os.MkdirAll(cacheDir, os.ModePerm); err != nil {
-		return nil, fmt.Errorf("creating a cache directory: %w", err)
-	}
-
+func New(cacheDir string) (*Cache, error) {
 	return &Cache{cacheDir: cacheDir}, nil
 }
 

pkg/cache/cache_test.go

@@ -11,7 +11,10 @@ import (
 )
 
 func setup(t *testing.T) (cache *Cache, teardown func()) {
-	cache, err := New("test-cache")
+	cacheDir := "test-cache"
+	assert.NoError(t, os.MkdirAll(cacheDir, os.ModePerm))
+
+	cache, err := New(cacheDir)
 	require.NoError(t, err)
 
 	return cache, func() {

pkg/combustion/registry.go

@@ -207,6 +207,11 @@ func registryArtefactsPath(ctx *image.Context) string {
 }
 
 func populateRegistry(ctx *image.Context, images []string) error {
+	imageCacheDir := filepath.Join(ctx.CacheDir, "images")
+	if err := os.MkdirAll(imageCacheDir, os.ModePerm); err != nil {
+		return fmt.Errorf("creating container image cache dir: %w", err)
+	}
+
 	bar := progressbar.Default(int64(len(images)), "Populating Embedded Artifact Registry...")
 	zap.S().Infof("Adding the following images to the embedded artifact registry:\n%s", images)
 
@@ -227,18 +232,29 @@ func populateRegistry(ctx *image.Context, images []string) error {
 	arch := ctx.ImageDefinition.Image.Arch.Short()
 
 	for _, img := range images {
-		if err = storeImage(img, arch, logFile); err != nil {
-			return fmt.Errorf("adding image to registry store: %w", err)
-		}
-
 		convertedImage := strings.ReplaceAll(img, "/", "_")
 		convertedImageName := fmt.Sprintf("%s-%s", convertedImage, registryTarSuffix)
 
+		imageCacheLocation := filepath.Join(imageCacheDir, convertedImageName)
 		imageTarDest := filepath.Join(registryArtefactsPath(ctx), convertedImageName)
-		if err = generateRegistryTar(imageTarDest, logFile); err != nil {
-			return fmt.Errorf("generating registry store tarball: %w", err)
-		}
 
+		if fileio.FileExists(imageCacheLocation) {
+			if err = fileio.CopyFile(imageCacheLocation, imageTarDest, fileio.NonExecutablePerms); err != nil {
+				return fmt.Errorf("copying cached container image: %w", err)
+			}
+		} else {
+			if err = storeImage(img, arch, logFile); err != nil {
+				return fmt.Errorf("adding image to registry store: %w", err)
+			}
+
+			if err = generateRegistryTar(imageTarDest, logFile); err != nil {
+				return fmt.Errorf("generating registry store tarball: %w", err)
+			}
+
+			if err = fileio.CopyFile(imageTarDest, imageCacheLocation, fileio.NonExecutablePerms); err != nil {
+				return fmt.Errorf("copying container image to cache: %w", err)
+			}
+		}
 		if err = bar.Add(1); err != nil {
 			zap.S().Debugf("Error incrementing the progress bar: %s", err)
 		}

pkg/eib/eib.go

@@ -126,6 +126,12 @@ func appendHelm(ctx *image.Context) {
 }
 
 func buildCombustion(ctx *image.Context, rootDir string) (*combustion.Combustion, error) {
+	cacheDir := filepath.Join(rootDir, "cache")
+	if err := os.MkdirAll(cacheDir, os.ModePerm); err != nil {
+		return nil, fmt.Errorf("creating a cache directory: %w", err)
+	}
+	ctx.CacheDir = cacheDir
+
 	combustionHandler := &combustion.Combustion{
 		NetworkConfigGenerator:       network.ConfigGenerator{},
 		NetworkConfiguratorInstaller: network.ConfiguratorInstaller{},
@@ -157,7 +163,7 @@ func buildCombustion(ctx *image.Context, rootDir string) (*combustion.Combustion
 	}
 
 	if ctx.ImageDefinition.Kubernetes.Version != "" {
-		c, err := cache.New(rootDir)
+		c, err := cache.New(cacheDir)
 		if err != nil {
 			return nil, fmt.Errorf("initialising cache instance: %w", err)
 		}

pkg/fileio/file_io.go

@@ -130,3 +130,16 @@ func createFileWithPerms(dest string, perms os.FileMode) (*os.File, error) {
 
 	return file, nil
 }
+
+func FileExists(filePath string) bool {
+	if _, err := os.Stat(filePath); err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return false
+		}
+
+		zap.S().Warnf("Searching for file at (%s) failed: %s", filePath, err)
+		return false
+	}
+
+	return true
+}

pkg/image/context.go

@@ -20,6 +20,8 @@ type Context struct {
 	ImageDefinition *Definition
 	// ArtifactSources contains the information necessary for the deployment of external artifacts.
 	ArtifactSources *ArtifactSources
+	// CacheDir contains all of the artifacts that are cached for the build process.
+	CacheDir string
 }
 
 type ArtifactSources struct {