Added redis tests

Makefile

@@ -510,6 +510,7 @@ run-blackbox-cloud-ci: check-blackbox-prerequisites check-awslocal binary $(BATS
 	echo running cloud CI bats tests; \
 	$(BATS) $(BATS_FLAGS) test/blackbox/cloud_only.bats
 	$(BATS) $(BATS_FLAGS) test/blackbox/sync_cloud.bats
+	$(BATS) $(BATS_FLAGS) test/blackbox/redis_s3.bats
 
 .PHONY: run-blackbox-dedupe-nightly
 run-blackbox-dedupe-nightly: check-blackbox-prerequisites check-awslocal binary binary-minimal

examples/config-redis.json

@@ -0,0 +1,37 @@
+{
+    "distSpecVersion": "1.1.0",
+    "storage": {
+        "dedupe": true,
+        "gc": true,
+        "rootDirectory": "/tmp/zot",
+        "cacheDriver": {
+            "name": "redis",
+            "rootDir": "/tmp/zot/_redis",
+            "url": "redis://localhost:6379"
+        },
+        "storageDriver": {
+            "name": "s3",
+            "rootdirectory": "/zot",
+            "region": "us-east-2",
+            "regionendpoint": "localhost:4566",
+            "bucket": "zot-storage",
+            "secure": false,
+            "skipverify": false
+        }
+    },
+    "http": {
+        "address": "0.0.0.0",
+        "port": "8484"
+    },
+    "log": {
+        "level": "debug"
+    },
+    "extensions": {
+      "ui": {
+        "enable": true
+      },
+      "search": {
+        "enable": true
+      }
+    }
+}

go.mod

@@ -45,6 +45,7 @@ require (
 )
 
 require (
+	github.com/alicebob/miniredis/v2 v2.33.0
 	github.com/aquasecurity/trivy v0.53.0
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.1
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.1
@@ -58,6 +59,7 @@ require (
 	github.com/notaryproject/notation-go v1.1.1
 	github.com/opencontainers/distribution-spec/specs-go v0.0.0-20240201174943-0f98d91a0afe
 	github.com/project-zot/mockoidc v0.0.0-20240610203808-d69d9e02020a
+	github.com/redis/go-redis/v9 v9.5.1
 	github.com/sigstore/cosign/v2 v2.2.4
 	github.com/swaggo/http-swagger v1.3.4
 	github.com/zitadel/oidc v1.13.5
@@ -91,6 +93,7 @@ require (
 	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Microsoft/hcsshim v0.12.3 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
+	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
 	github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/aquasecurity/table v1.8.0 // indirect
@@ -209,6 +212,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
+	github.com/yuin/gopher-lua v1.1.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect
 	go.opentelemetry.io/otel/metric v1.27.0 // indirect

go.sum

@@ -471,6 +471,10 @@ github.com/briandowns/spinner v1.23.1 h1:t5fDPmScwUjozhDj4FA46p5acZWIPXYE30qW2Pt
 github.com/briandowns/spinner v1.23.1/go.mod h1:LaZeM4wm2Ywy6vO571mvhQNRcWfRUnXOs0RcKV0wYKM=
 github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
 github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=
@@ -1306,6 +1310,8 @@ github.com/protocolbuffers/txtpbfmt v0.0.0-20231025115547-084445ff1adf h1:014O62
 github.com/protocolbuffers/txtpbfmt v0.0.0-20231025115547-084445ff1adf/go.mod h1:jgxiZysxFPM+iWKwQwPR+y+Jvo54ARd4EisXxKYpB5c=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
+github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=

pkg/cli/server/root.go

@@ -262,7 +262,8 @@ func validateCacheConfig(cfg *config.Config, log zlog.Logger) error {
 		}
 
 		// unsupported cache driver
-		if cfg.Storage.CacheDriver["name"] != storageConstants.DynamoDBDriverName {
+		if cfg.Storage.CacheDriver["name"] != storageConstants.DynamoDBDriverName &&
+			cfg.Storage.CacheDriver["name"] != storageConstants.RedisDriverName {
 			log.Error().Err(zerr.ErrBadConfig).
 				Interface("cacheDriver", cfg.Storage.CacheDriver["name"]).Msg("invalid cache config, unsupported cache driver")
 
@@ -272,8 +273,8 @@ func validateCacheConfig(cfg *config.Config, log zlog.Logger) error {
 
 	if !cfg.Storage.RemoteCache && cfg.Storage.CacheDriver != nil {
 		log.Warn().Err(zerr.ErrBadConfig).Str("directory", cfg.Storage.RootDirectory).
-			Msg("invalid cache config, remoteCache set to false but cacheDriver config (remote caching) provided for directory" +
-				"will ignore and use local caching")
+			Msg("invalid cache config, remoteCache set to false but cacheDriver config (remote caching) provided for " +
+				"directory will ignore and use local caching")
 	}
 
 	// subpaths

pkg/meta/meta.go

@@ -2,26 +2,39 @@ package meta
 
 import (
 	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
+	"github.com/redis/go-redis/v9"
 	"go.etcd.io/bbolt"
 
 	"zotregistry.dev/zot/errors"
 	"zotregistry.dev/zot/pkg/api/config"
 	"zotregistry.dev/zot/pkg/log"
 	"zotregistry.dev/zot/pkg/meta/boltdb"
 	mdynamodb "zotregistry.dev/zot/pkg/meta/dynamodb"
+	"zotregistry.dev/zot/pkg/meta/redisdb"
 	mTypes "zotregistry.dev/zot/pkg/meta/types"
+	sconstants "zotregistry.dev/zot/pkg/storage/constants"
 )
 
 func New(storageConfig config.StorageConfig, log log.Logger) (mTypes.MetaDB, error) {
 	if storageConfig.RemoteCache {
-		dynamoParams := getDynamoParams(storageConfig.CacheDriver, log)
+		if storageConfig.CacheDriver["name"] == sconstants.DynamoDBDriverName {
+			dynamoParams := getDynamoParams(storageConfig.CacheDriver, log)
 
-		client, err := mdynamodb.GetDynamoClient(dynamoParams)
-		if err != nil {
+			client, err := mdynamodb.GetDynamoClient(dynamoParams)
+			if err != nil {
+				return nil, err
+			}
+
+			return Create(sconstants.DynamoDBDriverName, client, dynamoParams, log) //nolint:contextcheck
+		}
+		// go-redis supports connecting via the redis uri specification (more convenient than parameter parsing)
+		redisURL := getRedisURL(storageConfig.CacheDriver, log)
+		client, err := redisdb.GetRedisClient(redisURL)
+		if err != nil { //nolint:wsl
 			return nil, err
 		}
 
-		return Create("dynamodb", client, dynamoParams, log) //nolint:contextcheck
+		return Create(sconstants.RedisDriverName, client, &redisdb.RedisDB{Client: client}, log) //nolint:contextcheck
 	}
 
 	params := boltdb.DBParameters{}
@@ -51,6 +64,18 @@ func Create(dbtype string, dbDriver, parameters interface{}, log log.Logger, //n
 
 			return boltdb.New(properDriver, log)
 		}
+	case "redis":
+		{
+			properDriver, ok := dbDriver.(*redis.Client)
+			if !ok {
+				log.Error().Err(errors.ErrTypeAssertionFailed).
+					Msgf("failed to cast type, expected type '%T' but got '%T'", &redis.Client{}, dbDriver)
+
+				return nil, errors.ErrTypeAssertionFailed
+			}
+
+			return redisdb.New(properDriver, log)
+		}
 	case "dynamodb":
 		{
 			properDriver, ok := dbDriver.(*dynamodb.Client)
@@ -122,6 +147,16 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) m
 	}
 }
 
+func getRedisURL(cacheDriverConfig map[string]interface{}, log log.Logger) string {
+	url, ok := toStringIfOk(cacheDriverConfig, "url", log)
+
+	if !ok {
+		log.Panic().Msg("redis parameters are not specified correctly, can't proceed")
+	}
+
+	return url
+}
+
 func toStringIfOk(cacheDriverConfig map[string]interface{}, param string, log log.Logger) (string, bool) {
 	val, ok := cacheDriverConfig[param]