[confmap/provider/secretsmanagerprovider] Add support for JSON secrets (

open-telemetry#32861)

**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
- Fixes `invalid memory address or nil pointer dereference` error when I
included this component in `opentelemetry-lambda/collector` lambda
layer.
- Fixes open-telemetry#32143 AWS Secrets Manager - JSON Secret Support

**Link to tracking Issue:** open-telemetry#32143 

**Testing:** Added unit tests. Manually tested in AWS Lambda Layer with
opentelemetry-lambda

**Documentation:** Update changelog and secretsmanagerprovider README.

---------

Co-authored-by: Antoine Toulme <[email protected]>
Co-authored-by: Evan Bradley <[email protected]>

.chloggen/fix-secrets-manager-confmap-provider.yaml

@@ -0,0 +1,30 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: confmap/provider/secretsmanagerprovider
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add support for JSON formatted secrets in secretsmanagerprovider confmap
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [32143]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  The `secretsmanagerprovider` confmap will now allow to get secret by a json key if the secret value is json.
+  To specify key separate key from secret name/arn by `#` e.g. `mySecret#mySecretKey`.
+
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

confmap/provider/secretsmanagerprovider/README.md

@@ -5,7 +5,8 @@ Collector the ability to read data stored in AWS Secrets Manager.
 ## How it works
 - Just use the placeholders with the following pattern `${secretsmanager:<arn or name>}`
 - Make sure you have the `secretsmanager:GetSecretValue` in the OTEL Collector Role
+- If your secret is a json string, you can get the value for a json key using the following pattern `${secretsmanager:<arn or name>#json-key}`
 
 Prerequisites:
-- Need to setup access keys from IAM console (aws_access_key_id and aws_secret_access_key) with permission to access Amazon Secrets Manager
-- For details, can take a look at https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/
+- Need to set up access keys from IAM console (aws_access_key_id and aws_secret_access_key) with permission to access Amazon Secrets Manager
+- For details, can take a look at https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/

confmap/provider/secretsmanagerprovider/go.mod

@@ -3,16 +3,25 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/confmap/provide
 go 1.21.0
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.27.0
+	github.com/aws/aws-sdk-go-v2/config v1.27.13
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.29.1
-	github.com/aws/smithy-go v1.20.2
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/confmap v0.102.2-0.20240606174409-6888f8f7a45f
 )
 
 require (
+	github.com/aws/aws-sdk-go-v2 v1.27.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.13 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.7 // indirect
+	github.com/aws/smithy-go v1.20.2 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect
 	github.com/knadh/koanf v1.5.0 // indirect

confmap/provider/secretsmanagerprovider/provider.go

@@ -5,19 +5,25 @@ package secretsmanagerprovider // import "github.com/open-telemetry/opentelemetr
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
+	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
 	"go.opentelemetry.io/collector/confmap"
 )
 
+type secretsManagerClient interface {
+	GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error)
+}
+
 const (
 	schemeName = "secretsmanager"
 )
 
 type provider struct {
-	client *secretsmanager.Client
+	client secretsManagerClient
 }
 
 // NewFactory returns a new confmap.ProviderFactory that creates a confmap.Provider
@@ -40,29 +46,56 @@ func newWithSettings(_ confmap.ProviderSettings) confmap.Provider {
 //
 // Deprecated: [v0.100.0] Use NewFactory() instead.
 func New() confmap.Provider {
-	return &provider{}
+	return &provider{client: nil}
 }
 
 func (provider *provider) Retrieve(ctx context.Context, uri string, _ confmap.WatcherFunc) (*confmap.Retrieved, error) {
 	if !strings.HasPrefix(uri, schemeName+":") {
 		return nil, fmt.Errorf("%q uri is not supported by %q provider", uri, schemeName)
 	}
 
-	secretArn := strings.Replace(uri, schemeName+":", "", 1)
+	// initialize the secrets manager client in the first call of Retrieve
+	if provider.client == nil {
+		cfg, err := config.LoadDefaultConfig(ctx)
+
+		if err != nil {
+			return nil, fmt.Errorf("failed to load configurations to initialize an AWS SDK client, error: %w", err)
+		}
+
+		provider.client = secretsmanager.NewFromConfig(cfg)
+	}
+
+	// Remove schemeName and split by # to get the json key
+	secretArn, secretJSONKey, jsonKeyFound := strings.Cut(strings.Replace(uri, schemeName+":", "", 1), "#")
 
 	input := &secretsmanager.GetSecretValueInput{
 		SecretId: &secretArn,
 	}
 
 	response, err := provider.client.GetSecretValue(ctx, input)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("error gtting secret: %w", err)
 	}
 
 	if response.SecretString == nil {
 		return nil, nil
 	}
 
+	if jsonKeyFound {
+		var secretFieldsMap map[string]any
+		err := json.Unmarshal([]byte(*response.SecretString), &secretFieldsMap)
+		if err != nil {
+			return nil, fmt.Errorf("error unmarshalling secret string: %w", err)
+		}
+
+		secretValue, ok := secretFieldsMap[secretJSONKey]
+		if !ok {
+			return nil, fmt.Errorf("field %q not found in secret map", secretJSONKey)
+		}
+
+		return confmap.NewRetrieved(secretValue)
+	}
+
 	return confmap.NewRetrieved(*response.SecretString)
 }
 

confmap/provider/secretsmanagerprovider/provider_test.go

@@ -5,67 +5,55 @@ package secretsmanagerprovider
 
 import (
 	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
+	"fmt"
 	"testing"
-	"time"
 
-	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
-	transport "github.com/aws/smithy-go/endpoints"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/confmap"
 )
 
-type resolver struct {
-	url string
+// Mock AWS secretsmanager
+type testSecretManagerClient struct {
+	secretValue string
 }
 
-func (r resolver) ResolveEndpoint(ctx context.Context, params secretsmanager.EndpointParameters) (transport.Endpoint, error) {
-	region := "us-east-1"
-	params.Region = &region
-	params.Endpoint = &r.url
-
-	old := secretsmanager.NewDefaultEndpointResolverV2()
-	return old.ResolveEndpoint(ctx, params)
+// Implement GetSecretValue()
+func (client *testSecretManagerClient) GetSecretValue(_ context.Context, _ *secretsmanager.GetSecretValueInput,
+	_ ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) {
+	return &secretsmanager.GetSecretValueOutput{SecretString: &client.secretValue}, nil
 }
 
-// Create a provider mocking s3provider works in normal cases
-func NewTestProvider(url string) confmap.Provider {
-	cfg := aws.NewConfig()
-
-	return &provider{client: secretsmanager.NewFromConfig(*cfg, secretsmanager.WithEndpointResolverV2(resolver{url: url}))}
+// Create a provider using mock secretsmanager client
+func NewTestProvider(secretValue string) confmap.Provider {
+	return &provider{client: &testSecretManagerClient{secretValue: secretValue}}
 }
 
 func TestSecretsManagerFetchSecret(t *testing.T) {
 	secretName := "FOO"
 	secretValue := "BAR"
 
-	s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
-		if request.Header.Get("X-Amz-Target") == "secretsmanager.GetSecretValue" {
-			response := &struct {
-				Arn          string `json:"ARN"`
-				CreatedDate  int64  `json:"CreatedDate"`
-				Name         string `json:"Name"`
-				SecretString string `json:"SecretString"`
-			}{
-				Arn:          secretName,
-				CreatedDate:  time.Now().Unix(),
-				Name:         secretName,
-				SecretString: secretValue,
-			}
-
-			b, _ := json.Marshal(response)
-			_, err := writer.Write(b)
-			require.NoError(t, err)
-			writer.WriteHeader(http.StatusOK)
-		}
-	}))
-	defer s.Close()
-	fp := NewTestProvider(s.URL)
+	fp := NewTestProvider(secretValue)
+	result, err := fp.Retrieve(context.Background(), "secretsmanager:"+secretName, nil)
+
+	assert.NoError(t, err)
+	assert.NoError(t, fp.Shutdown(context.Background()))
+
+	value, err := result.AsRaw()
+	assert.NoError(t, err)
+	assert.NotNil(t, value)
+	assert.Equal(t, secretValue, value)
+}
+
+func TestFetchSecretsManagerFieldValidJson(t *testing.T) {
+	secretName := "FOO#field1"
+	secretValue := "BAR"
+	secretJSON := fmt.Sprintf("{\"field1\": \"%s\"}", secretValue)
+
+	fp := NewTestProvider(secretJSON)
 	result, err := fp.Retrieve(context.Background(), "secretsmanager:"+secretName, nil)
+
 	assert.NoError(t, err)
 	assert.NoError(t, fp.Shutdown(context.Background()))
 
@@ -75,6 +63,29 @@ func TestSecretsManagerFetchSecret(t *testing.T) {
 	assert.Equal(t, secretValue, value)
 }
 
+func TestFetchSecretsManagerFieldInvalidJson(t *testing.T) {
+	secretName := "FOO#field1"
+	secretValue := "BAR"
+
+	fp := NewTestProvider(secretValue)
+	_, err := fp.Retrieve(context.Background(), "secretsmanager:"+secretName, nil)
+
+	assert.Error(t, err)
+	assert.NoError(t, fp.Shutdown(context.Background()))
+}
+
+func TestFetchSecretsManagerFieldMissingInJson(t *testing.T) {
+	secretName := "FOO#field1"
+	secretValue := "BAR"
+	secretJSON := fmt.Sprintf("{\"field0\": \"%s\"}", secretValue)
+
+	fp := NewTestProvider(secretJSON)
+	_, err := fp.Retrieve(context.Background(), "secretsmanager:"+secretName, nil)
+
+	assert.Error(t, err)
+	assert.NoError(t, fp.Shutdown(context.Background()))
+}
+
 func TestFactory(t *testing.T) {
 	p := NewFactory().Create(confmap.ProviderSettings{})
 	_, ok := p.(*provider)