added cache command & improved TAR file generation

CHANGELOG.md

@@ -1,4 +1,8 @@
 # Release Notes
 
-## Version 3.0 - 03/16/2020
+## Version 1.0.1 - 06/26/2020
+* Added cache command to list and clean the containerd docker images cache
+* Generated TAR only contains the extracted images for the chart, instead of all the images contained in the cache
+
+## Version 1.0 - 06/20/2020
 * First delivery on Github.

cmd/cache.go

@@ -0,0 +1,119 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"github.com/containerd/containerd/namespaces"
+	"github.com/gemalto/helm-image/internal/containerd"
+	"github.com/spf13/cobra"
+	"io"
+	"log"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+)
+
+type cacheCmd struct {
+	debug   bool
+	verbose bool
+}
+
+func newCacheListCmd(out io.Writer, c *cacheCmd) *cobra.Command {
+	return &cobra.Command{
+		Use:          "list",
+		Short:        "list docker images from local cache",
+		Long:         "list docker images from local cache",
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.list()
+		},
+	}
+}
+
+func newCacheCleanCmd(out io.Writer, c *cacheCmd) *cobra.Command {
+	return &cobra.Command{
+		Use:          "clean",
+		Short:        "remove all docker images from local cache",
+		Long:         "remove all docker images from local cache",
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return c.clean()
+		},
+	}
+}
+
+func newCacheCmd(out io.Writer) *cobra.Command {
+	c := &cacheCmd{}
+
+	cmd := &cobra.Command{
+		Use:          "cache",
+		Short:        "manage local cache of docker images",
+		Long:         "manage local cache of docker images",
+		SilenceUsage: true,
+	}
+
+	cmd.AddCommand(
+		newCacheListCmd(out, c),
+		newCacheCleanCmd(out, c),
+	)
+
+	cmd.PersistentFlags().BoolVarP(&c.verbose, "verbose", "v", false, "enable verbose output")
+
+	// When called through helm, debug mode is transmitted through the HELM_DEBUG envvar
+	helmDebug := os.Getenv("HELM_DEBUG")
+	if helmDebug == "1" || strings.EqualFold(helmDebug, "true") || strings.EqualFold(helmDebug, "on") {
+		c.debug = true
+	}
+
+	return cmd
+}
+
+func (c *cacheCmd) list() error {
+	serverStarted := make(chan bool)
+	serverKill := make(chan bool)
+	serverKilled := make(chan bool)
+	go containerd.Server(serverStarted, serverKill, serverKilled, c.debug)
+	if !<-serverStarted {
+		return fmt.Errorf("cannot start containerd server")
+	}
+	interrupt := make(chan os.Signal)
+	signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-interrupt
+		if c.debug {
+			log.Println("Sending interrupt signal to containerd server...")
+		}
+		serverKill <- true
+		<-serverKilled
+	}()
+	client, err := containerd.Client(c.debug)
+	if err != nil {
+		if c.debug {
+			log.Println("Sending interrupt signal to containerd server...")
+		}
+		serverKill <- true
+		<-serverKilled
+		return err
+	}
+	ctx := namespaces.WithNamespace(context.Background(), "default")
+	err = containerd.ListImages(ctx, client)
+	if err != nil {
+		if c.debug {
+			log.Println("Sending interrupt signal to containerd server...")
+		}
+		serverKill <- true
+		<-serverKilled
+		return err
+	}
+	if c.debug {
+		log.Println("Sending interrupt signal to containerd server...")
+	}
+	serverKill <- true
+	<-serverKilled
+	return nil
+}
+
+func (c *cacheCmd) clean() error {
+	return containerd.DeleteContainerdDirectories()
+}

cmd/list.go

@@ -107,7 +107,7 @@ func newListCmd(out io.Writer) *cobra.Command {
 	flags.StringArrayVar(&l.valuesOpts.Values, "set", []string{}, "set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	flags.StringArrayVar(&l.valuesOpts.StringValues, "set-string", []string{}, "set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	flags.StringArrayVar(&l.valuesOpts.FileValues, "set-file", []string{}, "set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)")
-	flags.BoolVar(&l.verbose, "verbose", false, "enable verbose output")
+	flags.BoolVarP(&l.verbose, "verbose", "v", false, "enable verbose output")
 
 	// When called through helm, debug mode is transmitted through the HELM_DEBUG envvar
 	helmDebug := os.Getenv("HELM_DEBUG")

cmd/root.go

@@ -14,6 +14,7 @@ func NewRootCmd(out io.Writer, args []string) *cobra.Command {
 	cmd.AddCommand(
 		newListCmd(out),
 		newSaveCmd(out),
+		newCacheCmd(out),
 	)
 	return cmd
 }

cmd/save.go

@@ -50,7 +50,7 @@ func newSaveCmd(out io.Writer) *cobra.Command {
 	flags.StringArrayVar(&s.valuesOpts.Values, "set", []string{}, "set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	flags.StringArrayVar(&s.valuesOpts.StringValues, "set-string", []string{}, "set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	flags.StringArrayVar(&s.valuesOpts.FileValues, "set-file", []string{}, "set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)")
-	flags.BoolVar(&s.verbose, "verbose", false, "enable verbose output")
+	flags.BoolVarP(&s.verbose, "verbose", "v", false, "enable verbose output")
 
 	// When called through helm, debug mode is transmitted through the HELM_DEBUG envvar
 	helmDebug := os.Getenv("HELM_DEBUG")
@@ -77,6 +77,19 @@ func (s *saveCmd) save() error {
 		debug:      s.debug,
 	}
 	images, err := l.list()
+	includedImagesMap := map[string]struct{}{}
+	for _, image := range images {
+		includedImagesMap[image] = struct{}{}
+		for _, excludedImage := range s.excludes {
+			if image == excludedImage {
+				delete(includedImagesMap, image)
+			}
+		}
+	}
+	var includedImages []string
+	for image := range includedImagesMap {
+		includedImages = append(includedImages, image)
+	}
 	if err != nil {
 		return err
 	}
@@ -85,10 +98,6 @@ func (s *saveCmd) save() error {
 	if err != nil {
 		return err
 	}
-	err = containerd.CreateContainerdDirectories()
-	if err != nil {
-		return err
-	}
 	serverStarted := make(chan bool)
 	serverKill := make(chan bool)
 	serverKilled := make(chan bool)
@@ -101,15 +110,15 @@ func (s *saveCmd) save() error {
 	go func() {
 		<-interrupt
 		if l.debug {
-			log.Println("Sending signal to containerd...")
+			log.Println("Sending interrupt signal to containerd server...")
 		}
 		serverKill <- true
 		<-serverKilled
 	}()
 	client, err := containerd.Client(l.debug)
 	if err != nil {
 		if l.debug {
-			log.Println("Sending signal to containerd...")
+			log.Println("Sending interrupt signal to containerd server...")
 		}
 		serverKill <- true
 		<-serverKilled
@@ -119,38 +128,28 @@ func (s *saveCmd) save() error {
 	for _, auth := range s.auths {
 		registry.AddAuthRegistry(auth)
 	}
-	for _, image := range images {
-		excluded := false
-		for _, excludedImage := range s.excludes {
-			if image == excludedImage {
-				excluded = true
-				break
-			}
-		}
-		if !excluded {
-			err = containerd.PullImage(ctx, client, registry.ConsoleCredentials, image, l.debug)
-			if err != nil {
-				if l.debug {
-					log.Println("Sending signal to containerd...")
-				}
-				serverKill <- true
-				<-serverKilled
-				return err
+	for _, image := range includedImages {
+		err = containerd.PullImage(ctx, client, registry.ConsoleCredentials, image, l.debug)
+		if err != nil {
+			if l.debug {
+				log.Println("Sending interrupt signal to containerd server...")
 			}
-			//}
+			serverKill <- true
+			<-serverKilled
+			return err
 		}
 	}
-	err = containerd.SaveImage(ctx, client, "", chart.Name()+".tar")
+	err = containerd.SaveImages(ctx, client, includedImages, chart.Name()+".tar")
 	if err != nil {
 		if l.debug {
-			log.Println("Sending signal to containerd...")
+			log.Println("Sending interrupt signal to containerd server...")
 		}
 		serverKill <- true
 		<-serverKilled
 		return err
 	}
 	if l.debug {
-		log.Println("Sending signal to containerd...")
+		log.Println("Sending interrupt signal to containerd server...")
 	}
 	serverKill <- true
 	<-serverKilled

internal/containerd/containerd.go

@@ -378,16 +378,24 @@ func closeClient(client *containerd.Client) {
 	}
 }
 
-func PullImage(ctx context.Context, client *containerd.Client, credentials registry.Credentials, imageName string, verbose bool) error {
-	fmt.Printf("Pulling image %s...\n", imageName)
-
+func imageRef(imageName string) (reference.Named, error) {
 	imageRef, err := reference.ParseNormalizedNamed(imageName)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	if reference.IsNameOnly(imageRef) {
 		imageRef = reference.TagNameOnly(imageRef)
 	}
+	return imageRef, nil
+}
+
+func PullImage(ctx context.Context, client *containerd.Client, credentials registry.Credentials, imageName string, verbose bool) error {
+	fmt.Printf("Pulling image %s...\n", imageName)
+
+	imageRef, err := imageRef(imageName)
+	if err != nil {
+		return err
+	}
 
 	resolver := docker.NewResolver(docker.ResolverOptions{
 		Tracker: docker.NewInMemoryTracker(),
@@ -456,30 +464,24 @@ func PullImage(ctx context.Context, client *containerd.Client, credentials regis
 	return nil
 }
 
-func SaveImage(ctx context.Context, client *containerd.Client, imageName string, fileName string) error {
-	if len(imageName) > 0 {
-		fmt.Printf("Saving image %s in %s...\n", imageName, fileName)
-	} else {
-		fmt.Printf("Saving all images in %s...\n", fileName)
+func SaveImages(ctx context.Context, client *containerd.Client, images []string, fileName string) error {
+	if len(images) == 0 {
+		return fmt.Errorf("no images to save")
 	}
+	fmt.Printf("Saving images in %s...\n", fileName)
 	var exportOpts []archive.ExportOpt
 	p, err := platforms.Parse("linux")
 	exportOpts = append(exportOpts, archive.WithPlatform(platforms.Ordered(p)))
-	var images []string
-	if len(imageName) > 0 {
-		images = append(images, imageName)
-	} else {
-		imgs, err := client.ListImages(ctx, "")
-		if err != nil {
-			return err
-		}
-		for _, img := range imgs {
-			images = append(images, img.Name())
-		}
+	if err != nil {
+		return err
 	}
 	is := client.ImageService()
 	for _, img := range images {
-		exportOpts = append(exportOpts, archive.WithImage(is, img))
+		imageRef, err := imageRef(img)
+		if err != nil {
+			return err
+		}
+		exportOpts = append(exportOpts, archive.WithImage(is, imageRef.String()))
 	}
 	f, err := os.Create(fileName)
 	if err != nil {
@@ -490,24 +492,20 @@ func SaveImage(ctx context.Context, client *containerd.Client, imageName string,
 	if err != nil {
 		return err
 	}
-	if len(imageName) > 0 {
-		fmt.Printf("Successfully saved image %s in %s\n", imageName, fileName)
-	} else {
-		fmt.Printf("Successfully saved all images in %s\n", fileName)
-	}
+	fmt.Printf("Successfully saved all images in %s\n", fileName)
 	return nil
 }
 
-//func ListImages(ctx context.Context, client *containerd.Client) error {
-//	imgs, err := client.ListImages(ctx, "")
-//	if err != nil {
-//		return err
-//	}
-//	for _, img := range imgs {
-//		fmt.Printf("%s\n", img.Name())
-//	}
-//	return nil
-//}
+func ListImages(ctx context.Context, client *containerd.Client) error {
+	imgs, err := client.ListImages(ctx, "")
+	if err != nil {
+		return err
+	}
+	for _, img := range imgs {
+		fmt.Printf("%s\n", img.Name())
+	}
+	return nil
+}
 
 func Client(debug bool) (*containerd.Client, error) {
 	homeDir, err := os.UserHomeDir()
@@ -540,6 +538,10 @@ func Client(debug bool) (*containerd.Client, error) {
 }
 
 func Server(serverStarted chan bool, serverKill chan bool, serverKilled chan bool, debug bool) {
+	err := CreateContainerdDirectories()
+	if err != nil {
+		log.Printf("Error: cannot create containerd directories: %s\n", err)
+	}
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
 		log.Printf("Error: cannot create containerd.log file: %s\n", err)
@@ -586,6 +588,16 @@ func Server(serverStarted chan bool, serverKill chan bool, serverKilled chan boo
 	serverKilled <- true
 }
 
+func DeleteContainerdDirectories() error {
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return err
+	}
+	baseDir := filepath.Join(homeDir, ".containerd")
+	err = os.RemoveAll(baseDir)
+	return err
+}
+
 func CreateContainerdDirectories() error {
 	homeDir, err := os.UserHomeDir()
 	if err != nil {

plugin.yaml

@@ -1,5 +1,5 @@
 name: "image"
-version: "1.0.0"
+version: "1.0.1"
 usage: "get docker images from a chart"
 description: "Helm plugin for getting docker images from a chart"
 command: "$HELM_PLUGIN_DIR/bin/helm-image"