S3: Support specifying S3 storage class (thanos-io#25)

Enable specifying S3 storage class

Allow specifying a storage class for
S3 object storage in
config.put_user_metadata:X-Amz-Storage-Class.
If not specified then STANDARD is used by default.

Solves: thanos-io/thanos#5663

Signed-off-by: Juan Rodriguez Hortala <[email protected]>

Signed-off-by: Juan Rodriguez Hortala <[email protected]>

Author: Juan Rodriguez Hortala

CHANGELOG.md

@@ -14,6 +14,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 ### Added
 - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support.
+- [#25](https://github.com/thanos-io/objstore/pull/25) S3: Support specifying S3 storage class.
 
 ### Changed
 

providers/s3/s3.go

@@ -96,6 +96,9 @@ const (
 	// NOTE: we're using a context value only because it's a very specific S3 option. If SSE will
 	// be available to wider set of backends we should probably add a variadic option to Get() and Upload().
 	sseConfigKey = ctxKey(0)
+
+	// Storage class header.
+	amzStorageClass = "X-Amz-Storage-Class"
 )
 
 var DefaultConfig = Config{
@@ -160,6 +163,7 @@ type Bucket struct {
 	client          *minio.Client
 	defaultSSE      encrypt.ServerSide
 	putUserMetadata map[string]string
+	storageClass    string
 	partSize        uint64
 	listObjectsV1   bool
 }
@@ -310,12 +314,23 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B
 		return nil, errors.Errorf("Initialize s3 client list objects version: Unsupported version %q was provided. Supported values are v1, v2", config.ListObjectsVersion)
 	}
 
+	var storageClass string
+	amzStorageClassLower := strings.ToLower(amzStorageClass)
+	for k, v := range config.PutUserMetadata {
+		if strings.ToLower(k) == amzStorageClassLower {
+			delete(config.PutUserMetadata, k)
+			storageClass = v
+			break
+		}
+	}
+
 	bkt := &Bucket{
 		logger:          logger,
 		name:            config.Bucket,
 		client:          client,
 		defaultSSE:      sse,
 		putUserMetadata: config.PutUserMetadata,
+		storageClass:    storageClass,
 		partSize:        config.PartSize,
 		listObjectsV1:   config.ListObjectsVersion == "v1",
 	}
@@ -486,6 +501,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error {
 			PartSize:             partSize,
 			ServerSideEncryption: sse,
 			UserMetadata:         b.putUserMetadata,
+			StorageClass:         b.storageClass,
 			// 4 is what minio-go have as the default. To be certain we do micro benchmark before any changes we
 			// ensure we pin this number to four.
 			// TODO(bwplotka): Consider adjusting this number to GOMAXPROCS or to expose this in config if it becomes bottleneck.

providers/s3/s3_test.go

@@ -419,3 +419,32 @@ func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) {
 	_, err = ioutil.ReadAll(reader)
 	testutil.Equals(t, io.ErrUnexpectedEOF, err)
 }
+
+func TestParseConfig_CustomStorageClass(t *testing.T) {
+	for _, testCase := range []struct{
+		name, storageClassKey string
+	}{
+		{ name: "ProperCase", storageClassKey: "X-Amz-Storage-Class" },
+		{ name: "UpperCase",  storageClassKey: "X-AMZ-STORAGE-CLASS" },
+		{ name: "LowerCase",  storageClassKey: "x-amz-storage-class" },
+		{ name: "MixedCase",  storageClassKey: "X-Amz-sToraGe-Class" },
+	}{
+		t.Run(testCase.name, func(t *testing.T) {
+			cfg := DefaultConfig
+			cfg.Endpoint = endpoint
+			storageClass := "STANDARD_IA"
+			cfg.PutUserMetadata[testCase.storageClassKey] = storageClass
+			bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test")
+			testutil.Ok(t, err)
+			testutil.Equals(t, storageClass, bkt.storageClass)
+		})
+	}
+}
+
+func TestParseConfig_DefaultStorageClassIsZero(t *testing.T) {
+	cfg := DefaultConfig
+	cfg.Endpoint = endpoint
+	bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test")
+	testutil.Ok(t, err)
+	testutil.Equals(t, "", bkt.storageClass)
+}