Store digest of latest image in ImagePolicy status

The new API field `.status.latestDigest` in the `ImagePolicy` kind
stores the digest of the image referred to by the the
`.status.latestImage` field.

This new field can be used to pin an image to an immutable descriptor
rather than to a potentially moving tag, increasing the security of
workloads deployed on a cluster.

The goal is to make use of the digest in IAC so that manifests can be
updated with the actual image digest.

This commit changes the format of the data stored in the caching
badger database from a list of strings to a list of `database.Tag`
objects where each tag carries a tag name and a digest value.

`ImageRepositoryReconciler` now fetched the digest of each image+tag
when it scans the registry for new tags. To accomplish this it issues
a HEAD request against the registry for each tag with the response
carrying the digest in the headers. Since this is a potentially
expensive operation involving network roundtrips for each tag, a
goroutine is spawned for each HEAD request to parallelize the fetching
process.

Migration from the old database format to the new one is taken care of
by the `badger.unmarshal` function which falls back to trying to
unmarshal the data into a string slice in case the attempt to
unmarshal it into a `database.Tag` slice fails. Subsequent `SetTags`
calls then store the data in the new format.

Because of its potential to significantly increase the amount of
network requests, the feature is disabled by default and can be
enabled using a feature flag for now.

closes #214

Signed-off-by: Max Jonas Werner <[email protected]>

api/v1beta2/imagepolicy_types.go

@@ -105,6 +105,10 @@ type ImagePolicyStatus struct {
 	// the image repository, when filtered and ordered according to
 	// the policy.
 	LatestImage string `json:"latestImage,omitempty"`
+	// LatestDigest is the digest of the latest image stored in the
+	// accompanying LatestImage field.
+	// +optional
+	LatestDigest string `json:"latestDigest,omitempty"`
 	// ObservedPreviousImage is the observed previous LatestImage. It is used
 	// to keep track of the previous and current images.
 	// +optional

config/crd/bases/image.toolkit.fluxcd.io_imagepolicies.yaml

@@ -383,6 +383,10 @@ spec:
                   - type
                   type: object
                 type: array
+              latestDigest:
+                description: LatestDigest is the digest of the latest image stored
+                  in the accompanying LatestImage field.
+                type: string
               latestImage:
                 description: LatestImage gives the first in the list of images scanned
                   by the image repository, when filtered and ordered according to

docs/api/image-reflector.md

@@ -313,6 +313,19 @@ the policy.</p>
 </tr>
 <tr>
 <td>
+<code>latestDigest</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>LatestDigest is the digest of the latest image stored in the
+accompanying LatestImage field.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>observedPreviousImage</code><br>
 <em>
 string

internal/controllers/controllers_fuzzer_test.go

@@ -50,7 +50,7 @@ import (
 	fuzz "github.com/AdaLogics/go-fuzz-headers"
 
 	imagev1 "github.com/fluxcd/image-reflector-controller/api/v1beta2"
-	"github.com/fluxcd/image-reflector-controller/internal/database"
+	ircbadger "github.com/fluxcd/image-reflector-controller/internal/database/badger"
 	"github.com/fluxcd/image-reflector-controller/internal/test"
 )
 
@@ -252,7 +252,7 @@ func initFunc() {
 
 	imageRepoReconciler = &ImageRepositoryReconciler{
 		Client:        k8sMgr.GetClient(),
-		Database:      database.NewBadgerDatabase(badgerDB),
+		Database:      ircbadger.NewBadgerDatabase(badgerDB),
 		EventRecorder: record.NewFakeRecorder(256),
 		patchOptions:  getPatchOptions(imageRepositoryOwnedConditions, "irc"),
 	}
@@ -263,7 +263,7 @@ func initFunc() {
 
 	imagePolicyReconciler = &ImagePolicyReconciler{
 		Client:        k8sMgr.GetClient(),
-		Database:      database.NewBadgerDatabase(badgerDB),
+		Database:      ircbadger.NewBadgerDatabase(badgerDB),
 		EventRecorder: record.NewFakeRecorder(256),
 		patchOptions:  getPatchOptions(imagePolicyOwnedConditions, "irc"),
 	}

internal/controllers/imagepolicy_controller.go

@@ -48,6 +48,7 @@ import (
 	pkgreconcile "github.com/fluxcd/pkg/runtime/reconcile"
 
 	imagev1 "github.com/fluxcd/image-reflector-controller/api/v1beta2"
+	"github.com/fluxcd/image-reflector-controller/internal/database"
 	"github.com/fluxcd/image-reflector-controller/internal/policy"
 )
 
@@ -108,7 +109,7 @@ type ImagePolicyReconciler struct {
 	helper.Metrics
 
 	ControllerName string
-	Database       DatabaseReader
+	Database       database.DatabaseReader
 	ACLOptions     acl.Options
 
 	patchOptions []patch.Option
@@ -259,6 +260,7 @@ func (r *ImagePolicyReconciler) reconcile(ctx context.Context, sp *patch.SerialP
 
 	// Cleanup the last result.
 	obj.Status.LatestImage = ""
+	obj.Status.LatestDigest = ""
 
 	// Get ImageRepository from reference.
 	repo, err := r.getImageRepository(ctx, obj)
@@ -317,7 +319,8 @@ func (r *ImagePolicyReconciler) reconcile(ctx context.Context, sp *patch.SerialP
 	}
 
 	// Write the observations on status.
-	obj.Status.LatestImage = repo.Spec.Image + ":" + latest
+	obj.Status.LatestImage = repo.Spec.Image + ":" + latest.Name
+	obj.Status.LatestDigest = latest.Digest
 	// If the old latest image and new latest image don't match, set the old
 	// image as the observed previous image.
 	// NOTE: The following allows the previous image to be set empty when
@@ -340,7 +343,7 @@ func (r *ImagePolicyReconciler) reconcile(ctx context.Context, sp *patch.SerialP
 	}
 
 	resultImage = repo.Spec.Image
-	resultTag = latest
+	resultTag = latest.Name
 
 	conditions.Delete(obj, meta.ReadyCondition)
 
@@ -386,36 +389,37 @@ func (r *ImagePolicyReconciler) getImageRepository(ctx context.Context, obj *ima
 
 // applyPolicy reads the tags of the given repository from the internal database
 // and applies the tag filters and constraints to return the latest image.
-func (r *ImagePolicyReconciler) applyPolicy(ctx context.Context, obj *imagev1.ImagePolicy, repo *imagev1.ImageRepository) (string, error) {
+func (r *ImagePolicyReconciler) applyPolicy(ctx context.Context, obj *imagev1.ImagePolicy, repo *imagev1.ImageRepository) (*database.Tag, error) {
 	policer, err := policy.PolicerFromSpec(obj.Spec.Policy)
 	if err != nil {
-		return "", errInvalidPolicy{err: fmt.Errorf("invalid policy: %w", err)}
+		return nil, errInvalidPolicy{err: fmt.Errorf("invalid policy: %w", err)}
 	}
 
 	// Read tags from database, apply and filter is configured and compute the
 	// result.
 	tags, err := r.Database.Tags(repo.Status.CanonicalImageName)
 	if err != nil {
-		return "", fmt.Errorf("failed to read tags from database: %w", err)
+		return nil, fmt.Errorf("failed to read tags from database: %w", err)
 	}
 
 	if len(tags) == 0 {
-		return "", errNoTagsInDatabase
+		return nil, errNoTagsInDatabase
 	}
 
 	// Apply tag filter.
 	if obj.Spec.FilterTags != nil {
 		filter, err := policy.NewRegexFilter(obj.Spec.FilterTags.Pattern, obj.Spec.FilterTags.Extract)
 		if err != nil {
-			return "", errInvalidPolicy{err: fmt.Errorf("failed to filter tags: %w", err)}
+			return nil, errInvalidPolicy{err: fmt.Errorf("failed to filter tags: %w", err)}
 		}
 		filter.Apply(tags)
-		tags = filter.Items()
-		latest, err := policer.Latest(tags)
+		tagNames := filter.Items()
+		latest, err := policer.Latest(tagNames)
 		if err != nil {
-			return "", err
+			return nil, err
 		}
-		return filter.GetOriginalTag(latest), nil
+		origTag := filter.GetOriginalTag(latest.Name)
+		return &origTag, nil
 	}
 	// Compute and return result.
 	return policer.Latest(tags)

internal/controllers/imagepolicy_controller_test.go

@@ -31,6 +31,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	imagev1 "github.com/fluxcd/image-reflector-controller/api/v1beta2"
+	"github.com/fluxcd/image-reflector-controller/internal/database"
 	"github.com/fluxcd/image-reflector-controller/internal/policy"
 )
 
@@ -231,7 +232,7 @@ func TestImagePolicyReconciler_applyPolicy(t *testing.T) {
 		filter     *imagev1.TagFilter
 		db         *mockDatabase
 		wantErr    bool
-		wantResult string
+		wantResult *database.Tag
 	}{
 		{
 			name:    "invalid policy",
@@ -251,16 +252,21 @@ func TestImagePolicyReconciler_applyPolicy(t *testing.T) {
 			wantErr: true,
 		},
 		{
-			name:       "semver, no tag filter",
-			policy:     imagev1.ImagePolicyChoice{SemVer: &imagev1.SemVerPolicy{Range: "1.0.x"}},
-			db:         &mockDatabase{TagData: []string{"1.0.0", "2.0.0", "1.0.1", "1.2.0"}},
-			wantResult: "1.0.1",
+			name:   "semver, no tag filter",
+			policy: imagev1.ImagePolicyChoice{SemVer: &imagev1.SemVerPolicy{Range: "1.0.x"}},
+			db: &mockDatabase{TagData: []database.Tag{
+				{Name: "1.0.0"},
+				{Name: "2.0.0"},
+				{Name: "1.0.1"},
+				{Name: "1.2.0"},
+			}},
+			wantResult: &database.Tag{Name: "1.0.1"},
 		},
 		{
 			name:    "invalid tag filter",
 			policy:  imagev1.ImagePolicyChoice{SemVer: &imagev1.SemVerPolicy{Range: "1.0.x"}},
 			filter:  &imagev1.TagFilter{Pattern: "[="},
-			db:      &mockDatabase{TagData: []string{"1.0.0", "1.0.1"}},
+			db:      &mockDatabase{TagData: []database.Tag{{Name: "1.0.0"}, {Name: "1.0.1"}}},
 			wantErr: true,
 		},
 		{
@@ -270,10 +276,14 @@ func TestImagePolicyReconciler_applyPolicy(t *testing.T) {
 				Pattern: "1.0.0-rc\\.(?P<num>[0-9]+)",
 				Extract: "$num",
 			},
-			db: &mockDatabase{TagData: []string{
-				"1.0.0", "1.0.0-rc.1", "1.0.0-rc.2", "1.0.0-rc.3", "1.0.1-rc.2",
+			db: &mockDatabase{TagData: []database.Tag{
+				{Name: "1.0.0"},
+				{Name: "1.0.0-rc.1"},
+				{Name: "1.0.0-rc.2"},
+				{Name: "1.0.0-rc.3"},
+				{Name: "1.0.1-rc.2"},
 			}},
-			wantResult: "1.0.0-rc.3",
+			wantResult: &database.Tag{Name: "1.0.0-rc.3"},
 		},
 		{
 			name:   "valid tag filter with alphabetical policy",
@@ -282,10 +292,14 @@ func TestImagePolicyReconciler_applyPolicy(t *testing.T) {
 				Pattern: "foo-(?P<word>[a-z]+)",
 				Extract: "$word",
 			},
-			db: &mockDatabase{TagData: []string{
-				"foo-aaa", "bar-bbb", "foo-zzz", "baz-nnn", "foo-ooo",
+			db: &mockDatabase{TagData: []database.Tag{
+				{Name: "foo-aaa"},
+				{Name: "bar-bbb"},
+				{Name: "foo-zzz"},
+				{Name: "baz-nnn"},
+				{Name: "foo-ooo"},
 			}},
-			wantResult: "foo-zzz",
+			wantResult: &database.Tag{Name: "foo-zzz"},
 		},
 	}
 

internal/controllers/imagerepository_controller.go

@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/google/go-containerregistry/pkg/authn"
@@ -54,6 +55,7 @@ import (
 	"github.com/fluxcd/pkg/runtime/reconcile"
 
 	imagev1 "github.com/fluxcd/image-reflector-controller/api/v1beta2"
+	"github.com/fluxcd/image-reflector-controller/internal/database"
 	"github.com/fluxcd/image-reflector-controller/internal/secret"
 )
 
@@ -109,10 +111,11 @@ type ImageRepositoryReconciler struct {
 
 	ControllerName string
 	Database       interface {
-		DatabaseWriter
-		DatabaseReader
+		database.DatabaseWriter
+		database.DatabaseReader
 	}
 	DeprecatedLoginOpts login.ProviderOptions
+	FetchDigests        bool
 
 	patchOptions []patch.Option
 }
@@ -516,8 +519,59 @@ func (r *ImageRepositoryReconciler) scan(ctx context.Context, obj *imagev1.Image
 		return 0, err
 	}
 
+	storedTags := make([]database.Tag, 0, len(filteredTags))
+
+	if r.FetchDigests {
+		resCh := make(chan database.Tag, len(filteredTags))
+		errCh := make(chan error, len(filteredTags))
+		var wg sync.WaitGroup
+
+		for _, tag := range filteredTags {
+			wg.Add(1)
+			go func(tag string, ch chan database.Tag) {
+				defer wg.Done()
+				res := database.Tag{
+					Name: tag,
+				}
+
+				ref := strings.Join([]string{ref.Context().Name(), tag}, ":")
+				tagRef, err := name.ParseReference(ref)
+				if err != nil {
+					errCh <- fmt.Errorf("failed parsing reference %q: %w", ref, err)
+					return
+				}
+				desc, err := remote.Head(tagRef, options...)
+				if err != nil {
+					errCh <- fmt.Errorf("failed fetching descriptor for %q: %w", tagRef.String(), err)
+					return
+				}
+				res.Digest = desc.Digest.String()
+
+				resCh <- res
+
+			}(tag, resCh)
+		}
+
+		wg.Wait()
+		close(resCh)
+		close(errCh)
+
+		for fetchErr := range errCh {
+			// return the first error we receive, all others are discarded
+			return 0, fetchErr
+		}
+
+		for t := range resCh {
+			storedTags = append(storedTags, t)
+		}
+	} else {
+		for _, tag := range filteredTags {
+			storedTags = append(storedTags, database.Tag{Name: tag})
+		}
+	}
+
 	canonicalName := ref.Context().String()
-	if err := r.Database.SetTags(canonicalName, filteredTags); err != nil {
+	if err := r.Database.SetTags(canonicalName, storedTags); err != nil {
 		return 0, fmt.Errorf("failed to set tags for %q: %w", canonicalName, err)
 	}