From 3fc9de146040aab3b90908a459b9f256a66602fb Mon Sep 17 00:00:00 2001 From: dbw7 Date: Sun, 21 Jul 2024 21:04:09 -0400 Subject: [PATCH] added image caching --- RELEASE_NOTES.md | 1 + pkg/cache/cache.go | 7 +----- pkg/combustion/combustion.go | 1 + pkg/combustion/registry.go | 46 +++++++++++++++++++++++++++++++----- pkg/eib/eib.go | 8 ++++++- pkg/image/context.go | 2 ++ 6 files changed, 52 insertions(+), 13 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d234614c..6c7eeede 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -9,6 +9,7 @@ * Artifact sources origin and metadata are now extracted from a configuration file (`config/artifacts.yaml`) * Dropped `-chart` suffix from installed Helm chart names * Added ability to build aarch64 images on an aarch64 host machine +* Added caching for container images ## API diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 21f1838d..d262e1ac 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -17,12 +17,7 @@ type Cache struct { cacheDir string } -func New(rootDir string) (*Cache, error) { - cacheDir := filepath.Join(rootDir, "cache") - if err := os.MkdirAll(cacheDir, os.ModePerm); err != nil { - return nil, fmt.Errorf("creating a cache directory: %w", err) - } - +func New(cacheDir string) (*Cache, error) { return &Cache{cacheDir: cacheDir}, nil } diff --git a/pkg/combustion/combustion.go b/pkg/combustion/combustion.go index 035d17fc..1b490dbf 100644 --- a/pkg/combustion/combustion.go +++ b/pkg/combustion/combustion.go @@ -62,6 +62,7 @@ type Combustion struct { RPMResolver rpmResolver RPMRepoCreator rpmRepoCreator Registry embeddedRegistry + CacheDir string } // Configure iterates over all separate Combustion components and configures them independently. diff --git a/pkg/combustion/registry.go b/pkg/combustion/registry.go index b13168dd..0c2c530b 100644 --- a/pkg/combustion/registry.go +++ b/pkg/combustion/registry.go @@ -2,6 +2,7 @@ package combustion import ( _ "embed" + "errors" "fmt" "io" "os" @@ -207,6 +208,11 @@ func registryArtefactsPath(ctx *image.Context) string { } func populateRegistry(ctx *image.Context, images []string) error { + imageCacheDir := filepath.Join(ctx.CacheDir, "images") + if err := os.MkdirAll(imageCacheDir, os.ModePerm); err != nil { + return fmt.Errorf("creating container image cache dir: %w", err) + } + bar := progressbar.Default(int64(len(images)), "Populating Embedded Artifact Registry...") zap.S().Infof("Adding the following images to the embedded artifact registry:\n%s", images) @@ -227,18 +233,34 @@ func populateRegistry(ctx *image.Context, images []string) error { arch := ctx.ImageDefinition.Image.Arch.Short() for _, img := range images { - if err = storeImage(img, arch, logFile); err != nil { - return fmt.Errorf("adding image to registry store: %w", err) - } - convertedImage := strings.ReplaceAll(img, "/", "_") convertedImageName := fmt.Sprintf("%s-%s", convertedImage, registryTarSuffix) + imageCacheLocation := filepath.Join(imageCacheDir, convertedImageName) imageTarDest := filepath.Join(registryArtefactsPath(ctx), convertedImageName) - if err = generateRegistryTar(imageTarDest, logFile); err != nil { - return fmt.Errorf("generating registry store tarball: %w", err) + + exists, err := imageExists(imageCacheLocation) + if err != nil { + return fmt.Errorf("checking if image already cached: %w", err) } + if exists { + if err = fileio.CopyFile(imageCacheLocation, imageTarDest, fileio.ExecutablePerms); err != nil { + return fmt.Errorf("copying cached container image: %w", err) + } + } else { + if err = storeImage(img, arch, logFile); err != nil { + return fmt.Errorf("adding image to registry store: %w", err) + } + + if err = generateRegistryTar(imageTarDest, logFile); err != nil { + return fmt.Errorf("generating registry store tarball: %w", err) + } + + if err = fileio.CopyFile(imageTarDest, imageCacheLocation, fileio.ExecutablePerms); err != nil { + return fmt.Errorf("copying container image to cache: %w", err) + } + } if err = bar.Add(1); err != nil { zap.S().Debugf("Error incrementing the progress bar: %s", err) } @@ -246,3 +268,15 @@ func populateRegistry(ctx *image.Context, images []string) error { return nil } + +func imageExists(imagePath string) (bool, error) { + if _, err := os.Stat(imagePath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + + return false, fmt.Errorf("looking for cached image: %w", err) + } + + return true, nil +} diff --git a/pkg/eib/eib.go b/pkg/eib/eib.go index a1092936..e6ef7838 100644 --- a/pkg/eib/eib.go +++ b/pkg/eib/eib.go @@ -126,6 +126,12 @@ func appendHelm(ctx *image.Context) { } func buildCombustion(ctx *image.Context, rootDir string) (*combustion.Combustion, error) { + cacheDir := filepath.Join(rootDir, "cache") + if err := os.MkdirAll(cacheDir, os.ModePerm); err != nil { + return nil, fmt.Errorf("creating a cache directory: %w", err) + } + ctx.CacheDir = cacheDir + combustionHandler := &combustion.Combustion{ NetworkConfigGenerator: network.ConfigGenerator{}, NetworkConfiguratorInstaller: network.ConfiguratorInstaller{}, @@ -157,7 +163,7 @@ func buildCombustion(ctx *image.Context, rootDir string) (*combustion.Combustion } if ctx.ImageDefinition.Kubernetes.Version != "" { - c, err := cache.New(rootDir) + c, err := cache.New(cacheDir) if err != nil { return nil, fmt.Errorf("initialising cache instance: %w", err) } diff --git a/pkg/image/context.go b/pkg/image/context.go index 4efac783..40190a27 100644 --- a/pkg/image/context.go +++ b/pkg/image/context.go @@ -20,6 +20,8 @@ type Context struct { ImageDefinition *Definition // ArtifactSources contains the information necessary for the deployment of external artifacts. ArtifactSources *ArtifactSources + // CacheDir contains all of the artifacts that are cached for the build process. + CacheDir string } type ArtifactSources struct {