diff --git a/CHANGELOG.md b/CHANGELOG.md index 23b820409f9..ab95ef2c047 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ * [FEATURE] Ingester: add experimental CLI flag `-ingester.ring.spread-minimizing-join-ring-in-order` that allows an ingester to register tokens in the ring only after all previous ingesters (with ID lower than its own ID) have already been registered. #5541 * [FEATURE] Ingester: add experimental support to compact the TSDB Head when the number of in-memory series is equal or greater than `-blocks-storage.tsdb.early-head-compaction-min-in-memory-series`, and the ingester estimates that the per-tenant TSDB Head compaction will reduce in-memory series by at least `-blocks-storage.tsdb.early-head-compaction-min-estimated-series-reduction-percentage`. #5371 * [FEATURE] Ingester: add new metrics for tracking native histograms in active series: `cortex_ingester_active_native_histogram_series`, `cortex_ingester_active_native_histogram_series_custom_tracker`, `cortex_ingester_active_native_histogram_buckets`, `cortex_ingester_active_native_histogram_buckets_custom_tracker`. The first 2 are the subsets of the existing and unmodified `cortex_ingester_active_series` and `cortex_ingester_active_series_custom_tracker` respectively, only tracking native histogram series, and the last 2 are the equivalents for tracking the number of buckets in native histogram series. #5318 +* [FEATURE] Add experimental CLI flag `-.s3.native-aws-auth-enabled` that allows to enable the default credentials provider chain of the AWS SDK. #5636 * [ENHANCEMENT] Overrides-exporter: Add new metrics for write path and alertmanager (`max_global_metadata_per_user`, `max_global_metadata_per_metric`, `request_rate`, `request_burst_size`, `alertmanager_notification_rate_limit`, `alertmanager_max_dispatcher_aggregation_groups`, `alertmanager_max_alerts_count`, `alertmanager_max_alerts_size_bytes`) and added flag `-overrides-exporter.enabled-metrics` to explicitly configure desired metrics, e.g. `-overrides-exporter.enabled-metrics=request_rate,ingestion_rate`. Default value for this flag is: `ingestion_rate,ingestion_burst_size,max_global_series_per_user,max_global_series_per_metric,max_global_exemplars_per_user,max_fetched_chunks_per_query,max_fetched_series_per_query,ruler_max_rules_per_rule_group,ruler_max_rule_groups_per_tenant`. #5376 * [ENHANCEMENT] Cardinality API: When zone aware replication is enabled, the label values cardinality API can now tolerate single zone failure #5178 * [ENHANCEMENT] Distributor: optimize sending requests to ingesters when incoming requests don't need to be modified. #5137 #5389 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 997d9e3d760..8646a463748 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -5057,6 +5057,17 @@ "fieldType": "string", "fieldCategory": "experimental" }, + { + "kind": "field", + "name": "native_aws_auth_enabled", + "required": false, + "desc": "If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "blocks-storage.s3.native-aws-auth-enabled", + "fieldType": "boolean", + "fieldCategory": "experimental" + }, { "kind": "block", "name": "sse", @@ -10576,6 +10587,17 @@ "fieldType": "string", "fieldCategory": "experimental" }, + { + "kind": "field", + "name": "native_aws_auth_enabled", + "required": false, + "desc": "If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "ruler-storage.s3.native-aws-auth-enabled", + "fieldType": "boolean", + "fieldCategory": "experimental" + }, { "kind": "block", "name": "sse", @@ -12542,6 +12564,17 @@ "fieldType": "string", "fieldCategory": "experimental" }, + { + "kind": "field", + "name": "native_aws_auth_enabled", + "required": false, + "desc": "If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "alertmanager-storage.s3.native-aws-auth-enabled", + "fieldType": "boolean", + "fieldCategory": "experimental" + }, { "kind": "block", "name": "sse", @@ -14787,6 +14820,17 @@ "fieldType": "string", "fieldCategory": "experimental" }, + { + "kind": "field", + "name": "native_aws_auth_enabled", + "required": false, + "desc": "If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "common.storage.s3.native-aws-auth-enabled", + "fieldType": "boolean", + "fieldCategory": "experimental" + }, { "kind": "block", "name": "sse", diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 064032743fb..7aa328ed1a1 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -49,6 +49,8 @@ Usage of ./cmd/mimir/mimir: Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit. (default 100) -alertmanager-storage.s3.max-idle-connections-per-host int Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100) + -alertmanager-storage.s3.native-aws-auth-enabled + [experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files. -alertmanager-storage.s3.region string S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it. -alertmanager-storage.s3.secret-access-key string @@ -667,6 +669,8 @@ Usage of ./cmd/mimir/mimir: Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit. (default 100) -blocks-storage.s3.max-idle-connections-per-host int Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100) + -blocks-storage.s3.native-aws-auth-enabled + [experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files. -blocks-storage.s3.region string S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it. -blocks-storage.s3.secret-access-key string @@ -821,6 +825,8 @@ Usage of ./cmd/mimir/mimir: Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit. (default 100) -common.storage.s3.max-idle-connections-per-host int Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100) + -common.storage.s3.native-aws-auth-enabled + [experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files. -common.storage.s3.region string S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it. -common.storage.s3.secret-access-key string @@ -2103,6 +2109,8 @@ Usage of ./cmd/mimir/mimir: Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit. (default 100) -ruler-storage.s3.max-idle-connections-per-host int Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used. (default 100) + -ruler-storage.s3.native-aws-auth-enabled + [experimental] If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files. -ruler-storage.s3.region string S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it. -ruler-storage.s3.secret-access-key string diff --git a/docs/sources/mimir/references/configuration-parameters/index.md b/docs/sources/mimir/references/configuration-parameters/index.md index b86a408cd33..dce31482a3d 100644 --- a/docs/sources/mimir/references/configuration-parameters/index.md +++ b/docs/sources/mimir/references/configuration-parameters/index.md @@ -4282,6 +4282,12 @@ The s3_backend block configures the connection to Amazon S3 object storage backe # CLI flag: -.s3.storage-class [storage_class: | default = ""] +# (experimental) If enabled, it will use the default authentication methods of +# the AWS SDK for go based on known environment variables and known AWS config +# files. +# CLI flag: -.s3.native-aws-auth-enabled +[native_aws_auth_enabled: | default = false] + sse: # Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3. # CLI flag: -.s3.sse.type diff --git a/pkg/storage/bucket/s3/bucket_client.go b/pkg/storage/bucket/s3/bucket_client.go index 3e98320f21e..792de710f17 100644 --- a/pkg/storage/bucket/s3/bucket_client.go +++ b/pkg/storage/bucket/s3/bucket_client.go @@ -59,6 +59,7 @@ func newS3Config(cfg Config) (s3.Config, error) { PutUserMetadata: putUserMetadata, SSEConfig: sseCfg, ListObjectsVersion: cfg.ListObjectsVersion, + AWSSDKAuth: cfg.NativeAWSAuthEnabled, HTTPConfig: s3.HTTPConfig{ IdleConnTimeout: model.Duration(cfg.HTTP.IdleConnTimeout), ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout), diff --git a/pkg/storage/bucket/s3/config.go b/pkg/storage/bucket/s3/config.go index 1d052c1f2d5..a7b3c2f2a87 100644 --- a/pkg/storage/bucket/s3/config.go +++ b/pkg/storage/bucket/s3/config.go @@ -75,15 +75,16 @@ func (cfg *HTTPConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { // Config holds the config options for an S3 backend type Config struct { - Endpoint string `yaml:"endpoint"` - Region string `yaml:"region"` - BucketName string `yaml:"bucket_name"` - SecretAccessKey flagext.Secret `yaml:"secret_access_key"` - AccessKeyID string `yaml:"access_key_id"` - Insecure bool `yaml:"insecure" category:"advanced"` - SignatureVersion string `yaml:"signature_version" category:"advanced"` - ListObjectsVersion string `yaml:"list_objects_version" category:"advanced"` - StorageClass string `yaml:"storage_class" category:"experimental"` + Endpoint string `yaml:"endpoint"` + Region string `yaml:"region"` + BucketName string `yaml:"bucket_name"` + SecretAccessKey flagext.Secret `yaml:"secret_access_key"` + AccessKeyID string `yaml:"access_key_id"` + Insecure bool `yaml:"insecure" category:"advanced"` + SignatureVersion string `yaml:"signature_version" category:"advanced"` + ListObjectsVersion string `yaml:"list_objects_version" category:"advanced"` + StorageClass string `yaml:"storage_class" category:"experimental"` + NativeAWSAuthEnabled bool `yaml:"native_aws_auth_enabled" category:"experimental"` SSE SSEConfig `yaml:"sse"` HTTP HTTPConfig `yaml:"http"` @@ -105,6 +106,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.StringVar(&cfg.SignatureVersion, prefix+"s3.signature-version", SignatureVersionV4, fmt.Sprintf("The signature version to use for authenticating against S3. Supported values are: %s.", strings.Join(supportedSignatureVersions, ", "))) f.StringVar(&cfg.ListObjectsVersion, prefix+"s3.list-objects-version", "", "Use a specific version of the S3 list object API. Supported values are v1 or v2. Default is unset.") f.StringVar(&cfg.StorageClass, prefix+"s3.storage-class", "", "The S3 storage class to use, not set by default. Details can be found at https://aws.amazon.com/s3/storage-classes/. Supported values are: "+strings.Join(supportedStorageClasses, ", ")) + f.BoolVar(&cfg.NativeAWSAuthEnabled, prefix+"s3.native-aws-auth-enabled", false, "If enabled, it will use the default authentication methods of the AWS SDK for go based on known environment variables and known AWS config files.") cfg.SSE.RegisterFlagsWithPrefix(prefix+"s3.sse.", f) cfg.HTTP.RegisterFlagsWithPrefix(prefix, f) } diff --git a/pkg/storage/bucket/s3/config_test.go b/pkg/storage/bucket/s3/config_test.go index b75be3a0e0e..84f81f3053f 100644 --- a/pkg/storage/bucket/s3/config_test.go +++ b/pkg/storage/bucket/s3/config_test.go @@ -109,6 +109,14 @@ func TestConfig_Validate(t *testing.T) { }, expected: errInvalidEndpointPrefix, }, + "should pass if native_aws_auth_enabled is set": { + setup: func() *Config { + return &Config{ + SignatureVersion: SignatureVersionV4, + NativeAWSAuthEnabled: true, + } + }, + }, } for testName, testData := range tests {