Skip to content

Commit eb14d38

Browse files
joelanfordjoelanford
authored
Containers image registry implementation with opt-in caching (#1626)
* containers/image registry implementation with layer caching Signed-off-by: Joe Lanford <[email protected]> * add containersimageregistry to standard registry test suite Signed-off-by: Joe Lanford <[email protected]> * add containers_image_openpgp build tag Signed-off-by: Joe Lanford <[email protected]> * containers/image registry: PR review follow-ups Signed-off-by: Joe Lanford <[email protected]> --------- Signed-off-by: Joe Lanford <[email protected]>
1 parent 2d1087f commit eb14d38

File tree

14 files changed

+652
-387
lines changed

14 files changed

+652
-387
lines changed

‎Makefile

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,14 @@ $(PROTOC):
2626
null :=
2727
space := $(null) #
2828
comma := ,
29-
# default to json1 for sqlite3
30-
TAGS := -tags=json1
29+
# default to json1 for sqlite3 and containers_image_openpgp for containers/image
30+
TAGS := json1,containers_image_openpgp
3131

3232
# Cluster to use for e2e testing
3333
CLUSTER ?= ""
3434
ifeq ($(CLUSTER), kind)
3535
# add kind to the list of tags
36-
TAGS += kind
37-
# convert tag format from space to comma list
38-
TAGS := $(subst $(space),$(comma),$(strip $(TAGS)))
36+
TAGS := $(TAGS),kind
3937
endif
4038

4139
# -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64 and windows/amd64
@@ -49,12 +47,12 @@ endif
4947
all: clean test build
5048

5149
$(CMDS):
52-
$(extra_env) $(GO) build $(extra_flags) $(TAGS) -o $@ ./cmd/$(notdir $@)
50+
$(extra_env) $(GO) build $(extra_flags) -tags=$(TAGS) -o $@ ./cmd/$(notdir $@)
5351

5452
.PHONY: $(OPM)
5553
$(OPM): opm_version_flags=-ldflags "-X '$(PKG)/cmd/opm/version.gitCommit=$(GIT_COMMIT)' -X '$(PKG)/cmd/opm/version.opmVersion=$(OPM_VERSION)' -X '$(PKG)/cmd/opm/version.buildDate=$(BUILD_DATE)'"
5654
$(OPM):
57-
$(extra_env) $(GO) build $(opm_version_flags) $(extra_flags) $(TAGS) -o $@ ./cmd/$(notdir $@)
55+
$(extra_env) $(GO) build $(opm_version_flags) $(extra_flags) -tags=$(TAGS) -o $@ ./cmd/$(notdir $@)
5856

5957
.PHONY: build
6058
build: clean $(CMDS) $(OPM)
@@ -63,8 +61,8 @@ build: clean $(CMDS) $(OPM)
6361
cross: opm_version_flags=-ldflags "-X '$(PKG)/cmd/opm/version.gitCommit=$(GIT_COMMIT)' -X '$(PKG)/cmd/opm/version.opmVersion=$(OPM_VERSION)' -X '$(PKG)/cmd/opm/version.buildDate=$(BUILD_DATE)'"
6462
cross:
6563
ifeq ($(shell go env GOARCH),amd64)
66-
GOOS=darwin CC=o64-clang CXX=o64-clang++ CGO_ENABLED=1 $(GO) build $(opm_version_flags) $(TAGS) -o "bin/darwin-amd64-opm" --ldflags "-extld=o64-clang" ./cmd/opm
67-
GOOS=windows CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CGO_ENABLED=1 $(GO) build $(opm_version_flags) $(TAGS) -o "bin/windows-amd64-opm" --ldflags "-extld=x86_64-w64-mingw32-gcc" -buildmode=exe ./cmd/opm
64+
GOOS=darwin CC=o64-clang CXX=o64-clang++ CGO_ENABLED=1 $(GO) build $(opm_version_flags) -tags=$(TAGS) -o "bin/darwin-amd64-opm" --ldflags "-extld=o64-clang" ./cmd/opm
65+
GOOS=windows CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CGO_ENABLED=1 $(GO) build $(opm_version_flags) -tags=$(TAGS) -o "bin/windows-amd64-opm" --ldflags "-extld=x86_64-w64-mingw32-gcc" -buildmode=exe ./cmd/opm
6866
endif
6967

7068
.PHONY: static
@@ -73,7 +71,7 @@ static: build
7371

7472
.PHONY: unit
7573
unit:
76-
$(GO) test -coverprofile=coverage.out $(SPECIFIC_UNIT_TEST) $(SPECIFIC_SKIP_UNIT_TEST) $(TAGS) $(TEST_RACE) -count=1 ./pkg/... ./alpha/...
74+
$(GO) test -coverprofile=coverage.out --coverpkg=./... $(SPECIFIC_UNIT_TEST) $(SPECIFIC_SKIP_UNIT_TEST) -tags=$(TAGS) $(TEST_RACE) -count=1 ./pkg/... ./alpha/...
7775

7876
.PHONY: tidy
7977
tidy:
@@ -102,10 +100,8 @@ image-upstream:
102100
docker build -f upstream-example.Dockerfile .
103101

104102
.PHONY: lint
105-
#lint:
106-
# find . -type f -name '*.go' ! -name '*.pb.go' -print0 | xargs -0 goimports -w
107103
lint: $(GOLANGCI_LINT)
108-
$(GOLANGCI_LINT) run $(GOLANGCI_LINT_ARGS)
104+
$(GOLANGCI_LINT) run --build-tags=$(TAGS) $(GOLANGCI_LINT_ARGS)
109105

110106
.PHONY: fix-lint
111107
fix-lint: $(GOLANGCI_LINT)
@@ -133,7 +129,7 @@ clean:
133129

134130
.PHONY: e2e
135131
e2e: $(GINKGO)
136-
$(GINKGO) --v --randomize-all --progress --trace --randomize-suites --race $(if $(TEST),-focus '$(TEST)') $(TAGS) ./test/e2e -- $(if $(SKIPTLS),-skip-tls-verify true) $(if $(USEHTTP),-use-http true)
132+
$(GINKGO) --v --randomize-all --progress --trace --randomize-suites --race $(if $(TEST),-focus '$(TEST)') -tags=$(TAGS) ./test/e2e -- $(if $(SKIPTLS),-skip-tls-verify true) $(if $(USEHTTP),-use-http true)
137133

138134
.PHONY: release
139135
export OPM_IMAGE_REPO ?= quay.io/operator-framework/opm

‎alpha/action/render.go

Lines changed: 2 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,8 @@ import (
2222
"github.com/operator-framework/operator-registry/alpha/property"
2323
"github.com/operator-framework/operator-registry/pkg/containertools"
2424
"github.com/operator-framework/operator-registry/pkg/image"
25-
"github.com/operator-framework/operator-registry/pkg/image/containerdregistry"
25+
"github.com/operator-framework/operator-registry/pkg/image/containersimageregistry"
2626
"github.com/operator-framework/operator-registry/pkg/lib/bundle"
27-
"github.com/operator-framework/operator-registry/pkg/lib/log"
2827
"github.com/operator-framework/operator-registry/pkg/registry"
2928
"github.com/operator-framework/operator-registry/pkg/sqlite"
3029
)
@@ -66,7 +65,7 @@ func (r Render) Run(ctx context.Context) (*declcfg.DeclarativeConfig, error) {
6665
logDeprecationMessage.Do(func() {})
6766
}
6867
if r.Registry == nil {
69-
reg, err := r.createRegistry()
68+
reg, err := containersimageregistry.NewDefault()
7069
if err != nil {
7170
return nil, fmt.Errorf("create registry: %v", err)
7271
}
@@ -101,26 +100,6 @@ func (r Render) Run(ctx context.Context) (*declcfg.DeclarativeConfig, error) {
101100
return combineConfigs(cfgs), nil
102101
}
103102

104-
func (r Render) createRegistry() (*containerdregistry.Registry, error) {
105-
cacheDir, err := os.MkdirTemp("", "render-registry-")
106-
if err != nil {
107-
return nil, fmt.Errorf("create tempdir: %v", err)
108-
}
109-
110-
reg, err := containerdregistry.NewRegistry(
111-
containerdregistry.WithCacheDir(cacheDir),
112-
113-
// The containerd registry impl is somewhat verbose, even on the happy path,
114-
// so discard all logger logs. Any important failures will be returned from
115-
// registry methods and eventually logged as fatal errors.
116-
containerdregistry.WithLog(log.Null()),
117-
)
118-
if err != nil {
119-
return nil, err
120-
}
121-
return reg, nil
122-
}
123-
124103
func (r Render) renderReference(ctx context.Context, ref string) (*declcfg.DeclarativeConfig, error) {
125104
stat, err := os.Stat(ref)
126105
if err != nil {

‎cmd/opm/alpha/bundle/validate.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010

1111
"github.com/operator-framework/operator-registry/pkg/containertools"
1212
"github.com/operator-framework/operator-registry/pkg/image"
13-
"github.com/operator-framework/operator-registry/pkg/image/containerdregistry"
13+
"github.com/operator-framework/operator-registry/pkg/image/containersimageregistry"
1414
"github.com/operator-framework/operator-registry/pkg/image/execregistry"
1515
"github.com/operator-framework/operator-registry/pkg/lib/bundle"
1616
)
@@ -69,7 +69,7 @@ func validateFunc(cmd *cobra.Command, _ []string) error {
6969
case containertools.PodmanTool, containertools.DockerTool:
7070
registry, err = execregistry.NewRegistry(tool, logger)
7171
case containertools.NoneTool:
72-
registry, err = containerdregistry.NewRegistry(containerdregistry.WithLog(logger))
72+
registry, err = containersimageregistry.NewDefault()
7373
default:
7474
err = fmt.Errorf("unrecognized container-tool option: %s", containerTool)
7575
}

‎cmd/opm/internal/util/util.go

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ import (
77

88
"github.com/spf13/cobra"
99

10-
"github.com/operator-framework/operator-registry/pkg/image/containerdregistry"
11-
"github.com/operator-framework/operator-registry/pkg/lib/log"
10+
"github.com/operator-framework/operator-registry/pkg/image"
11+
"github.com/operator-framework/operator-registry/pkg/image/containersimageregistry"
1212
)
1313

1414
// GetTLSOptions validates and returns TLS options set by opm flags
@@ -45,27 +45,15 @@ func GetTLSOptions(cmd *cobra.Command) (bool, bool, error) {
4545

4646
// This works in tandem with opm/index/cmd, which adds the relevant flags as persistent
4747
// as part of the root command (cmd/root/cmd) initialization
48-
func CreateCLIRegistry(cmd *cobra.Command) (*containerdregistry.Registry, error) {
48+
func CreateCLIRegistry(cmd *cobra.Command) (image.Registry, error) {
4949
skipTLSVerify, useHTTP, err := GetTLSOptions(cmd)
5050
if err != nil {
5151
return nil, err
5252
}
53-
54-
cacheDir, err := os.MkdirTemp("", "opm-registry-")
55-
if err != nil {
56-
return nil, err
57-
}
58-
59-
reg, err := containerdregistry.NewRegistry(
60-
containerdregistry.WithCacheDir(cacheDir),
61-
containerdregistry.SkipTLSVerify(skipTLSVerify),
62-
containerdregistry.WithPlainHTTP(useHTTP),
63-
containerdregistry.WithLog(log.Null()),
53+
return containersimageregistry.New(
54+
containersimageregistry.DefaultSystemContext,
55+
containersimageregistry.WithInsecureSkipTLSVerify(skipTLSVerify || useHTTP),
6456
)
65-
if err != nil {
66-
return nil, err
67-
}
68-
return reg, nil
6957
}
7058

7159
func OpenFileOrStdin(cmd *cobra.Command, args []string) (io.ReadCloser, string, error) {

‎go.mod

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ require (
2828
github.com/opencontainers/image-spec v1.1.1
2929
github.com/operator-framework/api v0.30.0
3030
github.com/otiai10/copy v1.14.1
31-
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
3231
github.com/pkg/errors v0.9.1
3332
github.com/sirupsen/logrus v1.9.3
3433
github.com/spf13/cobra v1.9.1
@@ -51,6 +50,7 @@ require (
5150
k8s.io/apimachinery v0.32.3
5251
k8s.io/client-go v0.32.3
5352
k8s.io/kubectl v0.32.0
53+
oras.land/oras-go/v2 v2.5.0
5454
sigs.k8s.io/controller-runtime v0.20.4
5555
sigs.k8s.io/kind v0.27.0
5656
sigs.k8s.io/yaml v1.4.0
@@ -65,10 +65,11 @@ require (
6565
github.com/MakeNowJust/heredoc v1.0.0 // indirect
6666
github.com/Microsoft/go-winio v0.6.2 // indirect
6767
github.com/Microsoft/hcsshim v0.12.9 // indirect
68+
github.com/VividCortex/ewma v1.2.0 // indirect
69+
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
6870
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
6971
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
7072
github.com/beorn7/perks v1.0.1 // indirect
71-
github.com/bshuster-repo/logrus-logstash-hook v1.0.0 // indirect
7273
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
7374
github.com/cespare/xxhash/v2 v2.3.0 // indirect
7475
github.com/containerd/cgroups/v3 v3.0.5 // indirect
@@ -82,7 +83,7 @@ require (
8283
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
8384
github.com/containers/ocicrypt v1.2.1 // indirect
8485
github.com/containers/storage v1.58.0 // indirect
85-
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
86+
github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect
8687
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
8788
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
8889
github.com/docker/distribution v2.8.3+incompatible // indirect
@@ -102,34 +103,47 @@ require (
102103
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
103104
github.com/go-logr/logr v1.4.2 // indirect
104105
github.com/go-logr/stdr v1.2.2 // indirect
106+
github.com/go-openapi/analysis v0.23.0 // indirect
107+
github.com/go-openapi/errors v0.22.1 // indirect
105108
github.com/go-openapi/jsonpointer v0.21.0 // indirect
106109
github.com/go-openapi/jsonreference v0.21.0 // indirect
110+
github.com/go-openapi/loads v0.22.0 // indirect
111+
github.com/go-openapi/runtime v0.28.0 // indirect
112+
github.com/go-openapi/spec v0.21.0 // indirect
113+
github.com/go-openapi/strfmt v0.23.0 // indirect
107114
github.com/go-openapi/swag v0.23.1 // indirect
115+
github.com/go-openapi/validate v0.24.0 // indirect
108116
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
109117
github.com/gogo/protobuf v1.3.2 // indirect
110118
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
111119
github.com/golang/protobuf v1.5.4 // indirect
112120
github.com/google/cel-go v0.22.1 // indirect
113121
github.com/google/gnostic-models v0.6.8 // indirect
122+
github.com/google/go-containerregistry v0.20.3 // indirect
114123
github.com/google/gofuzz v1.2.0 // indirect
115124
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
116125
github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
117126
github.com/google/uuid v1.6.0 // indirect
118127
github.com/gorilla/handlers v1.5.2 // indirect
119128
github.com/gorilla/mux v1.8.1 // indirect
120129
github.com/gorilla/websocket v1.5.0 // indirect
121-
github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect
130+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect
122131
github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect
123132
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
124133
github.com/inconshreveable/mousetrap v1.1.0 // indirect
125134
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
126135
github.com/josharian/intern v1.0.0 // indirect
127136
github.com/json-iterator/go v1.1.12 // indirect
128137
github.com/klauspost/compress v1.18.0 // indirect
138+
github.com/klauspost/pgzip v1.2.6 // indirect
139+
github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec // indirect
129140
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
130141
github.com/mailru/easyjson v0.9.0 // indirect
131142
github.com/mattn/go-isatty v0.0.20 // indirect
143+
github.com/mattn/go-runewidth v0.0.16 // indirect
144+
github.com/miekg/pkcs11 v1.1.1 // indirect
132145
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
146+
github.com/mitchellh/mapstructure v1.5.0 // indirect
133147
github.com/moby/locker v1.0.1 // indirect
134148
github.com/moby/spdystream v0.5.0 // indirect
135149
github.com/moby/sys/capability v0.4.0 // indirect
@@ -142,22 +156,37 @@ require (
142156
github.com/modern-go/reflect2 v1.0.2 // indirect
143157
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
144158
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
159+
github.com/oklog/ulid v1.3.1 // indirect
145160
github.com/opencontainers/runtime-spec v1.2.1 // indirect
146161
github.com/otiai10/mint v1.6.3 // indirect
147162
github.com/pelletier/go-toml v1.9.5 // indirect
148163
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
164+
github.com/proglottis/gpgme v0.1.4 // indirect
149165
github.com/prometheus/client_golang v1.21.1 // indirect
150166
github.com/prometheus/client_model v0.6.1 // indirect
151167
github.com/prometheus/common v0.62.0 // indirect
152168
github.com/prometheus/procfs v0.15.1 // indirect
153169
github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 // indirect
154170
github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 // indirect
155171
github.com/redis/go-redis/v9 v9.7.3 // indirect
172+
github.com/rivo/uniseg v0.4.7 // indirect
156173
github.com/russross/blackfriday/v2 v2.1.0 // indirect
174+
github.com/secure-systems-lab/go-securesystemslib v0.9.0 // indirect
175+
github.com/sigstore/fulcio v1.6.6 // indirect
176+
github.com/sigstore/protobuf-specs v0.4.1 // indirect
177+
github.com/sigstore/rekor v1.3.10 // indirect
178+
github.com/sigstore/sigstore v1.9.3 // indirect
179+
github.com/smallstep/pkcs7 v0.1.1 // indirect
157180
github.com/spiffe/go-spiffe/v2 v2.4.0 // indirect
181+
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect
158182
github.com/stoewer/go-strcase v1.3.0 // indirect
183+
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
184+
github.com/ulikunitz/xz v0.5.12 // indirect
185+
github.com/vbatts/tar-split v0.12.1 // indirect
186+
github.com/vbauerster/mpb/v8 v8.9.3 // indirect
159187
github.com/x448/float16 v0.8.4 // indirect
160188
github.com/zeebo/errs v1.3.0 // indirect
189+
go.mongodb.org/mongo-driver v1.14.0 // indirect
161190
go.opencensus.io v0.24.0 // indirect
162191
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
163192
go.opentelemetry.io/contrib/bridges/prometheus v0.57.0 // indirect
@@ -170,7 +199,7 @@ require (
170199
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 // indirect
171200
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect
172201
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0 // indirect
173-
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 // indirect
202+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 // indirect
174203
go.opentelemetry.io/otel/exporters/prometheus v0.54.0 // indirect
175204
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.8.0 // indirect
176205
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.32.0 // indirect
@@ -186,9 +215,9 @@ require (
186215
golang.org/x/crypto v0.37.0 // indirect
187216
golang.org/x/oauth2 v0.29.0 // indirect
188217
golang.org/x/term v0.31.0 // indirect
189-
golang.org/x/time v0.7.0 // indirect
218+
golang.org/x/time v0.11.0 // indirect
190219
golang.org/x/tools v0.31.0 // indirect
191-
google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
220+
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect
192221
google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect
193222
google.golang.org/genproto/googleapis/rpc v0.0.0-20250313205543-e70fdf4c4cb4 // indirect
194223
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect

0 commit comments

Comments
 (0)