From 54d2fa439a1b7765151ffb48a0debea7fd5902ec Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Wed, 2 Oct 2024 16:46:13 +0200 Subject: [PATCH] publish: Add ability to turn of layers manifest Add a new parameter to the publish command to turn on/off app layers manifest creation, posting, and inclusion its reference into the manifest of app being published. The layers manifest reference in a compose app manifest makes it non-OCI compliant and the vanilla registry rejects requests to post such the manifest. The new parameter allows turning off app layers manifest addition to the app manifest. As a result of it, a compose app manifest becomes OCI compliant so it can be pushed to the vanilla registry which is proved by the smoke test. The app layers manifest is redundant in case of `composectl` usage for app pulling since this utility retrieves meta about missing blobs from registry and it knows sizes of missing blobs (archived one). The app layers manifest is used by the older non composectl-based aklite versions to avoid writing C++ code for communicating with registries and retrieving blob metadata. Signed-off-by: Mike Sul --- cmd/composectl/cmd/publish.go | 15 +++++++------ internal/publish.go | 37 ++++++++++++++++---------------- pkg/compose/publish.go | 15 ++++++++----- test/fixtures/composectl_cmds.go | 8 +++++-- test/integration/smoke_test.go | 2 +- 5 files changed, 44 insertions(+), 33 deletions(-) diff --git a/cmd/composectl/cmd/publish.go b/cmd/composectl/cmd/publish.go index 4c05def..4d71810 100644 --- a/cmd/composectl/cmd/publish.go +++ b/cmd/composectl/cmd/publish.go @@ -28,11 +28,12 @@ var publishCmd = &cobra.Command{ type ( publishOptions struct { - ComposeFile string - DigestFile string - DryRun bool - PinnedImageURIs []string - LayersMetaFile string + ComposeFile string + DigestFile string + DryRun bool + PinnedImageURIs []string + LayersMetaFile string + CreateAppLayersManifest bool } ) @@ -43,6 +44,7 @@ func init() { publishCmd.Flags().BoolVar(&opts.DryRun, "dryrun", false, "Show what would be done, but don't actually publish") publishCmd.Flags().StringArrayVar(&opts.PinnedImageURIs, "pinned-images", nil, "A list of app images referred through digest URIs to pin app to") publishCmd.Flags().StringVarP(&opts.LayersMetaFile, "layers-meta", "l", "", "Json file containing App layers' metadata (size, usage)") + publishCmd.Flags().BoolVar(&opts.CreateAppLayersManifest, "layers-manifest", true, "Add app layers manifests to the app manifest") publishCmd.Run = func(cmd *cobra.Command, args []string) { fmt.Println(banner) @@ -84,5 +86,6 @@ func publishApp(cmd *cobra.Command, appRef *compose.AppRef, archList []string, o } } - DieNotNil(compose.DoPublish(cmd.Context(), appRef.Name, opts.ComposeFile, appRef.String(), opts.DigestFile, opts.DryRun, archList, pinnedImages, opts.LayersMetaFile)) + DieNotNil(compose.DoPublish(cmd.Context(), appRef.Name, opts.ComposeFile, appRef.String(), opts.DigestFile, + opts.DryRun, archList, pinnedImages, opts.LayersMetaFile, opts.CreateAppLayersManifest)) } diff --git a/internal/publish.go b/internal/publish.go index dd368b7..7c819cf 100644 --- a/internal/publish.go +++ b/internal/publish.go @@ -300,35 +300,34 @@ func CreateApp(ctx context.Context, config map[string]interface{}, target string if !ok { return "", fmt.Errorf("invalid manifest type, expected *ocischema.DeserializedManifest, got: %T", manifest) } - b, err := man.MarshalJSON() if err != nil { return "", err } - manMap := make(map[string]interface{}) - err = json.Unmarshal(b, &manMap) - if err != nil { - return "", err - } - - manMap["manifests"] = layerManifests - - b1, err := json.MarshalIndent(manMap, "", " ") - if err != nil { - return "", err - } + if layerManifests != nil { + manMap := make(map[string]interface{}) + err = json.Unmarshal(b, &manMap) + if err != nil { + return "", err + } - err = man.UnmarshalJSON(b1) - if err != nil { - return "", err + manMap["manifests"] = layerManifests + b, err = json.MarshalIndent(manMap, "", " ") + if err != nil { + return "", err + } + err = man.UnmarshalJSON(b) + if err != nil { + return "", err + } } - fmt.Printf(" |-> manifest size: %d\n", len(b1)) + fmt.Printf(" |-> manifest size: %d\n", len(b)) // TODO: this check is needed in order to overcome the aklite's check on the maximum manifest size (2048) // Once the new version of aklite is deployed (max manifest size = 16K) then this check can be removed or MaxArchNumb increased - if len(b1) >= MaxManifestBodySize { - return "", fmt.Errorf("app manifest size (%d) exceeds the maximum size limit (%d)", len(b1), MaxManifestBodySize) + if len(b) >= MaxManifestBodySize { + return "", fmt.Errorf("app manifest size (%d) exceeds the maximum size limit (%d)", len(b), MaxManifestBodySize) } svc, err := repo.Manifests(ctx, nil) if err != nil { diff --git a/pkg/compose/publish.go b/pkg/compose/publish.go index e36aef1..25fd568 100644 --- a/pkg/compose/publish.go +++ b/pkg/compose/publish.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "github.com/docker/distribution" "os" "strings" @@ -43,7 +44,8 @@ func loadProj(ctx context.Context, appName string, file string, content []byte) }) } -func DoPublish(ctx context.Context, appName string, file, target, digestFile string, dryRun bool, archList []string, pinnedImages map[string]digest.Digest, layersMetaFile string) error { +func DoPublish(ctx context.Context, appName string, file, target, digestFile string, dryRun bool, archList []string, + pinnedImages map[string]digest.Digest, layersMetaFile string, createAppLayersManifest bool) error { b, err := os.ReadFile(file) if err != nil { return err @@ -92,10 +94,13 @@ func DoPublish(ctx context.Context, appName string, file, target, digestFile str return fmt.Errorf("app cannot support more than %d architectures, found %d", internal.MaxArchNumb, len(appLayers)) } - fmt.Println("= Posting app layers manifests...") - layerManifests, err := PostAppLayersManifests(ctx, target, appLayers, dryRun) - if err != nil { - return err + var layerManifests []distribution.Descriptor + if createAppLayersManifest { + fmt.Println("= Posting app layers manifests...") + layerManifests, err = PostAppLayersManifests(ctx, target, appLayers, dryRun) + if err != nil { + return err + } } var appLayersMetaBytes []byte diff --git a/test/fixtures/composectl_cmds.go b/test/fixtures/composectl_cmds.go index 69b807d..d4c9de7 100644 --- a/test/fixtures/composectl_cmds.go +++ b/test/fixtures/composectl_cmds.go @@ -66,14 +66,18 @@ func NewApp(t *testing.T, composeDef string, options ...func(*App)) *App { return app } -func (a *App) Publish(t *testing.T) { +func (a *App) Publish(t *testing.T, dontPublishLayersManifest ...bool) { t.Run("publish app", func(t *testing.T) { digestFile := path.Join(a.Dir, "digest.sha256") tag, err := randomStringCrypto(7) if err != nil { t.Fatalf("failed to generate a random image tag value: %s\n", err) } - runCmd(t, a.Dir, "publish", "-d", digestFile, a.BaseUri+":"+tag, "amd64") + if len(dontPublishLayersManifest) > 0 && dontPublishLayersManifest[0] { + runCmd(t, a.Dir, "publish", "--layers-manifest=false", "-d", digestFile, a.BaseUri+":"+tag, "amd64") + } else { + runCmd(t, a.Dir, "publish", "-d", digestFile, a.BaseUri+":"+tag, "amd64") + } if b, err := os.ReadFile(digestFile); err == nil { a.PublishedUri = a.BaseUri + "@" + string(b) } else { diff --git a/test/integration/smoke_test.go b/test/integration/smoke_test.go index f549370..40fa56f 100644 --- a/test/integration/smoke_test.go +++ b/test/integration/smoke_test.go @@ -14,7 +14,7 @@ services: ` smokeTest := func(registry string, layersManifest bool) { app := fixtures.NewApp(t, appComposeDef, fixtures.WithRegistry(registry)) - app.Publish(t) + app.Publish(t, !layersManifest) app.Pull(t) defer app.Remove(t)