publish: Add ability to turn of layers manifest

Add a new parameter to the publish command to turn on/off app layers manifest creation, posting, and inclusion its reference into the manifest of app being published. The layers manifest reference in a compose app manifest makes it non-OCI compliant and the vanilla registry rejects requests to post such the manifest. The new parameter allows turning off app layers manifest addition to the app manifest. As a result of it, a compose app manifest becomes OCI compliant so it can be pushed to the vanilla registry which is proved by the smoke test. The app layers manifest is redundant in case of `composectl` usage for app pulling since this utility retrieves meta about missing blobs from registry and it knows sizes of missing blobs (archived one). The app layers manifest is used by the older non composectl-based aklite versions to avoid writing C++ code for communicating with registries and retrieving blob metadata. Signed-off-by: Mike Sul <[email protected]>
foundriesio · Oct 2, 2024 · 54d2fa4 · 54d2fa4
1 parent ba1f4e7
commit 54d2fa4
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 33 deletions.
diff --git a/cmd/composectl/cmd/publish.go b/cmd/composectl/cmd/publish.go
@@ -28,11 +28,12 @@ var publishCmd = &cobra.Command{
 
 type (
 	publishOptions struct {
-		ComposeFile     string
-		DigestFile      string
-		DryRun          bool
-		PinnedImageURIs []string
-		LayersMetaFile  string
+		ComposeFile             string
+		DigestFile              string
+		DryRun                  bool
+		PinnedImageURIs         []string
+		LayersMetaFile          string
+		CreateAppLayersManifest bool
 	}
 )
 
@@ -43,6 +44,7 @@ func init() {
 	publishCmd.Flags().BoolVar(&opts.DryRun, "dryrun", false, "Show what would be done, but don't actually publish")
 	publishCmd.Flags().StringArrayVar(&opts.PinnedImageURIs, "pinned-images", nil, "A list of app images referred through digest URIs to pin app to")
 	publishCmd.Flags().StringVarP(&opts.LayersMetaFile, "layers-meta", "l", "", "Json file containing App layers' metadata (size, usage)")
+	publishCmd.Flags().BoolVar(&opts.CreateAppLayersManifest, "layers-manifest", true, "Add app layers manifests to the app manifest")
 
 	publishCmd.Run = func(cmd *cobra.Command, args []string) {
 		fmt.Println(banner)
@@ -84,5 +86,6 @@ func publishApp(cmd *cobra.Command, appRef *compose.AppRef, archList []string, o
 		}
 	}
 
-	DieNotNil(compose.DoPublish(cmd.Context(), appRef.Name, opts.ComposeFile, appRef.String(), opts.DigestFile, opts.DryRun, archList, pinnedImages, opts.LayersMetaFile))
+	DieNotNil(compose.DoPublish(cmd.Context(), appRef.Name, opts.ComposeFile, appRef.String(), opts.DigestFile,
+		opts.DryRun, archList, pinnedImages, opts.LayersMetaFile, opts.CreateAppLayersManifest))
 }
diff --git a/internal/publish.go b/internal/publish.go
@@ -300,35 +300,34 @@ func CreateApp(ctx context.Context, config map[string]interface{}, target string
 	if !ok {
 		return "", fmt.Errorf("invalid manifest type, expected *ocischema.DeserializedManifest, got: %T", manifest)
 	}
-
 	b, err := man.MarshalJSON()
 	if err != nil {
 		return "", err
 	}
 
-	manMap := make(map[string]interface{})
-	err = json.Unmarshal(b, &manMap)
-	if err != nil {
-		return "", err
-	}
-
-	manMap["manifests"] = layerManifests
-
-	b1, err := json.MarshalIndent(manMap, "", "   ")
-	if err != nil {
-		return "", err
-	}
+	if layerManifests != nil {
+		manMap := make(map[string]interface{})
+		err = json.Unmarshal(b, &manMap)
+		if err != nil {
+			return "", err
+		}
 
-	err = man.UnmarshalJSON(b1)
-	if err != nil {
-		return "", err
+		manMap["manifests"] = layerManifests
+		b, err = json.MarshalIndent(manMap, "", "   ")
+		if err != nil {
+			return "", err
+		}
+		err = man.UnmarshalJSON(b)
+		if err != nil {
+			return "", err
+		}
 	}
 
-	fmt.Printf("  |-> manifest size: %d\n", len(b1))
+	fmt.Printf("  |-> manifest size: %d\n", len(b))
 	// TODO: this check is needed in order to overcome the aklite's check on the maximum manifest size (2048)
 	// Once the new version of aklite is deployed (max manifest size = 16K) then this check can be removed or MaxArchNumb increased
-	if len(b1) >= MaxManifestBodySize {
-		return "", fmt.Errorf("app manifest size (%d) exceeds the maximum size limit (%d)", len(b1), MaxManifestBodySize)
+	if len(b) >= MaxManifestBodySize {
+		return "", fmt.Errorf("app manifest size (%d) exceeds the maximum size limit (%d)", len(b), MaxManifestBodySize)
 	}
 	svc, err := repo.Manifests(ctx, nil)
 	if err != nil {

diff --git a/pkg/compose/publish.go b/pkg/compose/publish.go
@@ -6,6 +6,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/docker/distribution"
 	"os"
 	"strings"
 
@@ -43,7 +44,8 @@ func loadProj(ctx context.Context, appName string, file string, content []byte)
 	})
 }
 
-func DoPublish(ctx context.Context, appName string, file, target, digestFile string, dryRun bool, archList []string, pinnedImages map[string]digest.Digest, layersMetaFile string) error {
+func DoPublish(ctx context.Context, appName string, file, target, digestFile string, dryRun bool, archList []string,
+	pinnedImages map[string]digest.Digest, layersMetaFile string, createAppLayersManifest bool) error {
 	b, err := os.ReadFile(file)
 	if err != nil {
 		return err
@@ -92,10 +94,13 @@ func DoPublish(ctx context.Context, appName string, file, target, digestFile str
 		return fmt.Errorf("app cannot support more than %d architectures, found %d", internal.MaxArchNumb, len(appLayers))
 	}
 
-	fmt.Println("= Posting app layers manifests...")
-	layerManifests, err := PostAppLayersManifests(ctx, target, appLayers, dryRun)
-	if err != nil {
-		return err
+	var layerManifests []distribution.Descriptor
+	if createAppLayersManifest {
+		fmt.Println("= Posting app layers manifests...")
+		layerManifests, err = PostAppLayersManifests(ctx, target, appLayers, dryRun)
+		if err != nil {
+			return err
+		}
 	}
 
 	var appLayersMetaBytes []byte

diff --git a/test/fixtures/composectl_cmds.go b/test/fixtures/composectl_cmds.go
@@ -66,14 +66,18 @@ func NewApp(t *testing.T, composeDef string, options ...func(*App)) *App {
 	return app
 }
 
-func (a *App) Publish(t *testing.T) {
+func (a *App) Publish(t *testing.T, dontPublishLayersManifest ...bool) {
 	t.Run("publish app", func(t *testing.T) {
 		digestFile := path.Join(a.Dir, "digest.sha256")
 		tag, err := randomStringCrypto(7)
 		if err != nil {
 			t.Fatalf("failed to generate a random image tag value: %s\n", err)
 		}
-		runCmd(t, a.Dir, "publish", "-d", digestFile, a.BaseUri+":"+tag, "amd64")
+		if len(dontPublishLayersManifest) > 0 && dontPublishLayersManifest[0] {
+			runCmd(t, a.Dir, "publish", "--layers-manifest=false", "-d", digestFile, a.BaseUri+":"+tag, "amd64")
+		} else {
+			runCmd(t, a.Dir, "publish", "-d", digestFile, a.BaseUri+":"+tag, "amd64")
+		}
 		if b, err := os.ReadFile(digestFile); err == nil {
 			a.PublishedUri = a.BaseUri + "@" + string(b)
 		} else {

diff --git a/test/integration/smoke_test.go b/test/integration/smoke_test.go
@@ -14,7 +14,7 @@ services:
 `
 	smokeTest := func(registry string, layersManifest bool) {
 		app := fixtures.NewApp(t, appComposeDef, fixtures.WithRegistry(registry))
-		app.Publish(t)
+		app.Publish(t, !layersManifest)
 
 		app.Pull(t)
 		defer app.Remove(t)