Feat/support aws sdk default credential provider chain (#5636)

Signed-off-by: Aljoscha Poertner <[email protected]>

CHANGELOG.md

@@ -27,6 +27,7 @@
 * [FEATURE] Ingester: add experimental CLI flag `-ingester.ring.spread-minimizing-join-ring-in-order` that allows an ingester to register tokens in the ring only after all previous ingesters (with ID lower than its own ID) have already been registered. #5541
 * [FEATURE] Ingester: add experimental support to compact the TSDB Head when the number of in-memory series is equal or greater than `-blocks-storage.tsdb.early-head-compaction-min-in-memory-series`, and the ingester estimates that the per-tenant TSDB Head compaction will reduce in-memory series by at least `-blocks-storage.tsdb.early-head-compaction-min-estimated-series-reduction-percentage`. #5371
 * [FEATURE] Ingester: add new metrics for tracking native histograms in active series: `cortex_ingester_active_native_histogram_series`, `cortex_ingester_active_native_histogram_series_custom_tracker`, `cortex_ingester_active_native_histogram_buckets`, `cortex_ingester_active_native_histogram_buckets_custom_tracker`. The first 2 are the subsets of the existing and unmodified `cortex_ingester_active_series` and `cortex_ingester_active_series_custom_tracker` respectively, only tracking native histogram series, and the last 2 are the equivalents for tracking the number of buckets in native histogram series. #5318
+* [FEATURE] Add experimental CLI flag `-<prefix>.s3.native-aws-auth-enabled` that allows to enable the default credentials provider chain of the AWS SDK. #5636
 * [ENHANCEMENT] Overrides-exporter: Add new metrics for write path and alertmanager (`max_global_metadata_per_user`, `max_global_metadata_per_metric`, `request_rate`, `request_burst_size`, `alertmanager_notification_rate_limit`, `alertmanager_max_dispatcher_aggregation_groups`, `alertmanager_max_alerts_count`, `alertmanager_max_alerts_size_bytes`) and added flag `-overrides-exporter.enabled-metrics` to explicitly configure desired metrics, e.g. `-overrides-exporter.enabled-metrics=request_rate,ingestion_rate`. Default value for this flag is: `ingestion_rate,ingestion_burst_size,max_global_series_per_user,max_global_series_per_metric,max_global_exemplars_per_user,max_fetched_chunks_per_query,max_fetched_series_per_query,ruler_max_rules_per_rule_group,ruler_max_rule_groups_per_tenant`. #5376
 * [ENHANCEMENT] Cardinality API: When zone aware replication is enabled, the label values cardinality API can now tolerate single zone failure #5178
 * [ENHANCEMENT] Distributor: optimize sending requests to ingesters when incoming requests don't need to be modified. #5137 #5389

cmd/mimir/config-descriptor.json

@@ -5057,6 +5057,17 @@
               "fieldType": "string",
               "fieldCategory": "experimental"
             },
+            {
+              "kind": "field",
+              "name": "native_aws_auth_enabled",
+              "required": false,
+              "desc": "If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.",
+              "fieldValue": null,
+              "fieldDefaultValue": false,
+              "fieldFlag": "blocks-storage.s3.native-aws-auth-enabled",
+              "fieldType": "boolean",
+              "fieldCategory": "experimental"
+            },
             {
               "kind": "block",
               "name": "sse",
@@ -10576,6 +10587,17 @@
               "fieldType": "string",
               "fieldCategory": "experimental"
             },
+            {
+              "kind": "field",
+              "name": "native_aws_auth_enabled",
+              "required": false,
+              "desc": "If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.",
+              "fieldValue": null,
+              "fieldDefaultValue": false,
+              "fieldFlag": "ruler-storage.s3.native-aws-auth-enabled",
+              "fieldType": "boolean",
+              "fieldCategory": "experimental"
+            },
             {
               "kind": "block",
               "name": "sse",
@@ -12542,6 +12564,17 @@
               "fieldType": "string",
               "fieldCategory": "experimental"
             },
+            {
+              "kind": "field",
+              "name": "native_aws_auth_enabled",
+              "required": false,
+              "desc": "If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.",
+              "fieldValue": null,
+              "fieldDefaultValue": false,
+              "fieldFlag": "alertmanager-storage.s3.native-aws-auth-enabled",
+              "fieldType": "boolean",
+              "fieldCategory": "experimental"
+            },
             {
               "kind": "block",
               "name": "sse",
@@ -14787,6 +14820,17 @@
                   "fieldType": "string",
                   "fieldCategory": "experimental"
                 },
+                {
+                  "kind": "field",
+                  "name": "native_aws_auth_enabled",
+                  "required": false,
+                  "desc": "If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.",
+                  "fieldValue": null,
+                  "fieldDefaultValue": false,
+                  "fieldFlag": "common.storage.s3.native-aws-auth-enabled",
+                  "fieldType": "boolean",
+                  "fieldCategory": "experimental"
+                },
                 {
                   "kind": "block",
                   "name": "sse",

cmd/mimir/help-all.txt.tmpl

@@ -49,6 +49,8 @@ Usage of ./cmd/mimir/mimir:
     	Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit. (default 100)
   -alertmanager-storage.s3.max-idle-connections-per-host int
     	Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100)
+  -alertmanager-storage.s3.native-aws-auth-enabled
+    	[experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.
   -alertmanager-storage.s3.region string
     	S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it.
   -alertmanager-storage.s3.secret-access-key string
@@ -667,6 +669,8 @@ Usage of ./cmd/mimir/mimir:
     	Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit. (default 100)
   -blocks-storage.s3.max-idle-connections-per-host int
     	Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100)
+  -blocks-storage.s3.native-aws-auth-enabled
+    	[experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.
   -blocks-storage.s3.region string
     	S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it.
   -blocks-storage.s3.secret-access-key string
@@ -821,6 +825,8 @@ Usage of ./cmd/mimir/mimir:
     	Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit. (default 100)
   -common.storage.s3.max-idle-connections-per-host int
     	Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100)
+  -common.storage.s3.native-aws-auth-enabled
+    	[experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.
   -common.storage.s3.region string
     	S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it.
   -common.storage.s3.secret-access-key string
@@ -2103,6 +2109,8 @@ Usage of ./cmd/mimir/mimir:
     	Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit. (default 100)
   -ruler-storage.s3.max-idle-connections-per-host int
     	Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100)
+  -ruler-storage.s3.native-aws-auth-enabled
+    	[experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.
   -ruler-storage.s3.region string
     	S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it.
   -ruler-storage.s3.secret-access-key string

docs/sources/mimir/references/configuration-parameters/index.md

@@ -4282,6 +4282,12 @@ The s3_backend block configures the connection to Amazon S3 object storage backe
 # CLI flag: -<prefix>.s3.storage-class
 [storage_class: <string> | default = ""]
 
+# (experimental) If enabled, it will use the default authentication methods of
+# the AWS SDK for go based on known environment variables and known AWS config
+# files.
+# CLI flag: -<prefix>.s3.native-aws-auth-enabled
+[native_aws_auth_enabled: <boolean> | default = false]
+
 sse:
   # Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3.
   # CLI flag: -<prefix>.s3.sse.type

pkg/storage/bucket/s3/bucket_client.go

@@ -59,6 +59,7 @@ func newS3Config(cfg Config) (s3.Config, error) {
 		PutUserMetadata:    putUserMetadata,
 		SSEConfig:          sseCfg,
 		ListObjectsVersion: cfg.ListObjectsVersion,
+		AWSSDKAuth:         cfg.NativeAWSAuthEnabled,
 		HTTPConfig: s3.HTTPConfig{
 			IdleConnTimeout:       model.Duration(cfg.HTTP.IdleConnTimeout),
 			ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout),

pkg/storage/bucket/s3/config.go

@@ -75,15 +75,16 @@ func (cfg *HTTPConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 
 // Config holds the config options for an S3 backend
 type Config struct {
-	Endpoint           string         `yaml:"endpoint"`
-	Region             string         `yaml:"region"`
-	BucketName         string         `yaml:"bucket_name"`
-	SecretAccessKey    flagext.Secret `yaml:"secret_access_key"`
-	AccessKeyID        string         `yaml:"access_key_id"`
-	Insecure           bool           `yaml:"insecure" category:"advanced"`
-	SignatureVersion   string         `yaml:"signature_version" category:"advanced"`
-	ListObjectsVersion string         `yaml:"list_objects_version" category:"advanced"`
-	StorageClass       string         `yaml:"storage_class" category:"experimental"`
+	Endpoint             string         `yaml:"endpoint"`
+	Region               string         `yaml:"region"`
+	BucketName           string         `yaml:"bucket_name"`
+	SecretAccessKey      flagext.Secret `yaml:"secret_access_key"`
+	AccessKeyID          string         `yaml:"access_key_id"`
+	Insecure             bool           `yaml:"insecure" category:"advanced"`
+	SignatureVersion     string         `yaml:"signature_version" category:"advanced"`
+	ListObjectsVersion   string         `yaml:"list_objects_version" category:"advanced"`
+	StorageClass         string         `yaml:"storage_class" category:"experimental"`
+	NativeAWSAuthEnabled bool           `yaml:"native_aws_auth_enabled" category:"experimental"`
 
 	SSE  SSEConfig  `yaml:"sse"`
 	HTTP HTTPConfig `yaml:"http"`
@@ -105,6 +106,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.StringVar(&cfg.SignatureVersion, prefix+"s3.signature-version", SignatureVersionV4, fmt.Sprintf("The signature version to use for authenticating against S3. Supported values are: %s.", strings.Join(supportedSignatureVersions, ", ")))
 	f.StringVar(&cfg.ListObjectsVersion, prefix+"s3.list-objects-version", "", "Use a specific version of the S3 list object API. Supported values are v1 or v2. Default is unset.")
 	f.StringVar(&cfg.StorageClass, prefix+"s3.storage-class", "", "The S3 storage class to use, not set by default. Details can be found at https://aws.amazon.com/s3/storage-classes/. Supported values are: "+strings.Join(supportedStorageClasses, ", "))
+	f.BoolVar(&cfg.NativeAWSAuthEnabled, prefix+"s3.native-aws-auth-enabled", false, "If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.")
 	cfg.SSE.RegisterFlagsWithPrefix(prefix+"s3.sse.", f)
 	cfg.HTTP.RegisterFlagsWithPrefix(prefix, f)
 }

pkg/storage/bucket/s3/config_test.go

@@ -109,6 +109,14 @@ func TestConfig_Validate(t *testing.T) {
 			},
 			expected: errInvalidEndpointPrefix,
 		},
+		"should pass if native_aws_auth_enabled is set": {
+			setup: func() *Config {
+				return &Config{
+					SignatureVersion:     SignatureVersionV4,
+					NativeAWSAuthEnabled: true,
+				}
+			},
+		},
 	}
 
 	for testName, testData := range tests {