fix: use matching signing region for IAM join method (#47425)

integration/ec2_test.go

@@ -45,6 +45,7 @@ import (
 	"github.com/gravitational/teleport/lib/backend"
 	"github.com/gravitational/teleport/lib/backend/lite"
 	cloudimds "github.com/gravitational/teleport/lib/cloud/imds"
+	cloudaws "github.com/gravitational/teleport/lib/cloud/imds/aws"
 	"github.com/gravitational/teleport/lib/defaults"
 	"github.com/gravitational/teleport/lib/labels"
 	"github.com/gravitational/teleport/lib/service"
@@ -143,6 +144,12 @@ func getIID(ctx context.Context, t *testing.T) imds.InstanceIdentityDocument {
 func getCallerIdentity(ctx context.Context, t *testing.T) *sts.GetCallerIdentityOutput {
 	cfg, err := config.LoadDefaultConfig(ctx)
 	require.NoError(t, err)
+	if cfg.Region == "" {
+		imdsClient, err := cloudaws.NewInstanceMetadataClient(ctx)
+		require.NoError(t, err)
+		cfg.Region, err = imdsClient.GetRegion(ctx)
+		require.NoError(t, err, "trying to get local region from IMDSv2")
+	}
 	stsClient := sts.NewFromConfig(cfg)
 	output, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{})
 	require.NoError(t, err)

lib/auth/join/iam/endpoints.go

@@ -75,14 +75,24 @@ var (
 	// FIPSSTSEndpoints returns the set of known valid FIPS AWS STS endpoints.
 	FIPSSTSEndpoints = sync.OnceValue(func() []string {
 		return []string{
-			fipsSTSEndpointUSEast1,
+			"sts-fips.us-east-1.amazonaws.com",
 			"sts-fips.us-east-2.amazonaws.com",
 			"sts-fips.us-west-1.amazonaws.com",
 			"sts-fips.us-west-2.amazonaws.com",
 			"sts.us-gov-east-1.amazonaws.com",
 			"sts.us-gov-west-1.amazonaws.com",
 		}
 	})
-)
 
-const fipsSTSEndpointUSEast1 = "sts-fips.us-east-1.amazonaws.com"
+	// FIPSSTSRegions returns the set of known AWS regions with FIPS STS endpoints.
+	FIPSSTSRegions = sync.OnceValue(func() []string {
+		return []string{
+			"us-east-1",
+			"us-east-2",
+			"us-west-1",
+			"us-west-2",
+			"us-gov-east-1",
+			"us-gov-west-1",
+		}
+	})
+)

lib/auth/join/iam/iam.go

@@ -41,8 +41,8 @@ const (
 )
 
 type stsIdentityRequestOptions struct {
-	fipsEndpointOption aws.FIPSEndpointState
-	imdsClient         imdsClient
+	useFIPS    bool
+	imdsClient imdsClient
 }
 
 type stsIdentityRequestOption func(cfg *stsIdentityRequestOptions)
@@ -51,11 +51,7 @@ type stsIdentityRequestOption func(cfg *stsIdentityRequestOptions)
 // regions, this will use the us-east-1 FIPS endpoint.
 func WithFIPSEndpoint(useFIPS bool) stsIdentityRequestOption {
 	return func(opts *stsIdentityRequestOptions) {
-		if useFIPS {
-			opts.fipsEndpointOption = aws.FIPSEndpointStateEnabled
-		} else {
-			opts.fipsEndpointOption = aws.FIPSEndpointStateUnset
-		}
+		opts.useFIPS = useFIPS
 	}
 }
 
@@ -87,11 +83,30 @@ func CreateSignedSTSIdentityRequest(ctx context.Context, challenge string, opts
 		return nil, trace.Wrap(err, "loading default AWS config")
 	}
 
+	if awsConfig.Region == "" {
+		// We can try to get the local region from IMDSv2 if running on EC2.
+		region, err := getEC2LocalRegion(ctx, &options)
+		if err != nil {
+			slog.InfoContext(ctx, "Failed to resolve local AWS region from environment or IMDS. Using us-east-1 by default. This will fail in non-default AWS partitions. Consider setting AWS_REGION or enabling IMDSv2.",
+				slog.Any("error", err))
+			region = "us-east-1"
+		}
+		awsConfig.Region = region
+	}
+
+	if options.useFIPS && !slices.Contains(FIPSSTSRegions(), awsConfig.Region) {
+		slog.InfoContext(ctx, "AWS region does not have a FIPS STS endpoint, attempting to use us-east-1 instead. This will fail in non-default AWS partitions.",
+			slog.String("region", awsConfig.Region))
+		awsConfig.Region = "us-east-1"
+	}
+
 	var signedRequest bytes.Buffer
 	stsClient := sts.NewFromConfig(awsConfig,
-		sts.WithEndpointResolverV2(newCustomResolver(challenge, &options)),
+		sts.WithEndpointResolverV2(newCustomResolver(challenge)),
 		func(stsOpts *sts.Options) {
-			stsOpts.EndpointOptions.UseFIPSEndpoint = options.fipsEndpointOption
+			if options.useFIPS {
+				stsOpts.EndpointOptions.UseFIPSEndpoint = aws.FIPSEndpointStateEnabled
+			}
 			// Use a fake HTTP client to record the request.
 			stsOpts.HTTPClient = &httpRequestRecorder{&signedRequest}
 			// httpRequestRecorder intentionally records the request and returns
@@ -132,43 +147,20 @@ func getEC2LocalRegion(ctx context.Context, opts *stsIdentityRequestOptions) (st
 type customResolver struct {
 	defaultResolver sts.EndpointResolverV2
 	challenge       string
-	opts            *stsIdentityRequestOptions
 }
 
-func newCustomResolver(challenge string, opts *stsIdentityRequestOptions) *customResolver {
+func newCustomResolver(challenge string) *customResolver {
 	return &customResolver{
 		defaultResolver: sts.NewDefaultEndpointResolverV2(),
 		challenge:       challenge,
-		opts:            opts,
 	}
 }
 
 // ResolveEndpoint implements [sts.EndpointResolverV2].
 func (r customResolver) ResolveEndpoint(ctx context.Context, params sts.EndpointParameters) (smithyendpoints.Endpoint, error) {
-	if aws.ToString(params.Region) == "" {
-		// If we don't have a region from the environment here this will fail to
-		// resolve. We can try to get the local region from IMDSv2 if running on EC2.
-		region, err := getEC2LocalRegion(ctx, r.opts)
-		switch {
-		case trace.IsNotFound(err):
-			params.Region = aws.String("aws-global")
-			params.UseGlobalEndpoint = aws.Bool(true)
-		case err != nil:
-			return smithyendpoints.Endpoint{}, trace.Wrap(err, "failed to resolve local AWS region from environment or IMDS")
-		default:
-			params.Region = aws.String(region)
-		}
-	}
 	endpoint, err := r.defaultResolver.ResolveEndpoint(ctx, params)
 	if err != nil {
-		return smithyendpoints.Endpoint{}, trace.Wrap(err)
-	}
-	if aws.ToBool(params.UseFIPS) && !slices.Contains(FIPSSTSEndpoints(), endpoint.URI.Host) {
-		// The default resolver will return non-existent endpoints if FIPS was
-		// requested in regions outside the USA. Use the FIPS endpoint in
-		// us-east-1 instead.
-		slog.InfoContext(ctx, "The AWS SDK resolved an invalid FIPS STS endpoint, attempting to use the us-east-1 FIPS STS endpoint instead. This will fail in non-default AWS partitions.", "resolved", endpoint.URI.Host)
-		endpoint.URI.Host = fipsSTSEndpointUSEast1
+		return endpoint, trace.Wrap(err)
 	}
 	// Add challenge as a header to be signed.
 	endpoint.Headers.Add(challengeHeaderKey, r.challenge)

lib/auth/join/iam/iam_test.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/gravitational/teleport/lib/auth/join/iam"
@@ -40,51 +41,61 @@ func TestCreateSignedSTSIdentityRequest(t *testing.T) {
 	const challenge = "asdf12345"
 
 	for desc, tc := range map[string]struct {
-		envRegion      string
-		imdsRegion     string
-		fips           bool
-		expectEndpoint string
-		expectError    string
+		envRegion             string
+		imdsRegion            string
+		fips                  bool
+		expectError           string
+		expectEndpoint        string
+		expectSignatureRegion string
 	}{
 		"no region": {
-			expectEndpoint: "sts.amazonaws.com",
+			expectEndpoint:        "sts.us-east-1.amazonaws.com",
+			expectSignatureRegion: "us-east-1",
 		},
 		"no region fips": {
-			fips:           true,
-			expectEndpoint: "sts-fips.us-east-1.amazonaws.com",
+			fips:                  true,
+			expectEndpoint:        "sts-fips.us-east-1.amazonaws.com",
+			expectSignatureRegion: "us-east-1",
 		},
 		"us-west-2": {
-			envRegion:      "us-west-2",
-			expectEndpoint: "sts.us-west-2.amazonaws.com",
+			envRegion:             "us-west-2",
+			expectEndpoint:        "sts.us-west-2.amazonaws.com",
+			expectSignatureRegion: "us-west-2",
 		},
 		"us-west-2 with region from imdsv2": {
-			imdsRegion:     "us-west-2",
-			expectEndpoint: "sts.us-west-2.amazonaws.com",
+			imdsRegion:            "us-west-2",
+			expectEndpoint:        "sts.us-west-2.amazonaws.com",
+			expectSignatureRegion: "us-west-2",
 		},
 		"us-west-2 fips": {
-			envRegion:      "us-west-2",
-			fips:           true,
-			expectEndpoint: "sts-fips.us-west-2.amazonaws.com",
+			envRegion:             "us-west-2",
+			fips:                  true,
+			expectEndpoint:        "sts-fips.us-west-2.amazonaws.com",
+			expectSignatureRegion: "us-west-2",
 		},
 		"us-west-2 fips with region from imdsv2": {
-			imdsRegion:     "us-west-2",
-			fips:           true,
-			expectEndpoint: "sts-fips.us-west-2.amazonaws.com",
+			imdsRegion:            "us-west-2",
+			fips:                  true,
+			expectEndpoint:        "sts-fips.us-west-2.amazonaws.com",
+			expectSignatureRegion: "us-west-2",
 		},
 		"eu-central-1": {
-			envRegion:      "eu-central-1",
-			expectEndpoint: "sts.eu-central-1.amazonaws.com",
+			envRegion:             "eu-central-1",
+			expectEndpoint:        "sts.eu-central-1.amazonaws.com",
+			expectSignatureRegion: "eu-central-1",
 		},
 		"eu-central-1 fips": {
 			envRegion: "eu-central-1",
 			fips:      true,
 			// All non-US regions have no FIPS endpoint and use the FIPS
 			// endpoint in us-east-1.
-			expectEndpoint: "sts-fips.us-east-1.amazonaws.com",
+			expectEndpoint:        "sts-fips.us-east-1.amazonaws.com",
+			expectSignatureRegion: "us-east-1",
 		},
 		"ap-southeast-1": {
-			envRegion:      "ap-southeast-1",
-			expectEndpoint: "sts.ap-southeast-1.amazonaws.com",
+			envRegion:             "ap-southeast-1",
+			expectEndpoint:        "sts.ap-southeast-1.amazonaws.com",
+			expectSignatureRegion: "ap-southeast-1",
 		},
 		"ap-southeast-1 fips": {
 			envRegion: "ap-southeast-1",
@@ -95,17 +106,20 @@ func TestCreateSignedSTSIdentityRequest(t *testing.T) {
 			// recognized by STS in the default partition. It will fail when
 			// Auth sends the request to AWS, but this unit test only exercises
 			// the client-side request generation.
-			expectEndpoint: "sts-fips.us-east-1.amazonaws.com",
+			expectEndpoint:        "sts-fips.us-east-1.amazonaws.com",
+			expectSignatureRegion: "us-east-1",
 		},
 		"govcloud": {
-			envRegion:      "us-gov-east-1",
-			expectEndpoint: "sts.us-gov-east-1.amazonaws.com",
+			envRegion:             "us-gov-east-1",
+			expectEndpoint:        "sts.us-gov-east-1.amazonaws.com",
+			expectSignatureRegion: "us-gov-east-1",
 		},
 		"govcloud fips": {
 			envRegion: "us-gov-east-1",
 			fips:      true,
 			// All govcloud endpoints are FIPS.
-			expectEndpoint: "sts.us-gov-east-1.amazonaws.com",
+			expectEndpoint:        "sts.us-gov-east-1.amazonaws.com",
+			expectSignatureRegion: "us-gov-east-1",
 		},
 	} {
 		t.Run(desc, func(t *testing.T) {
@@ -132,8 +146,8 @@ func TestCreateSignedSTSIdentityRequest(t *testing.T) {
 				iam.WithFIPSEndpoint(tc.fips),
 				iam.WithIMDSClient(imdsClient))
 			if tc.expectError != "" {
-				require.Error(t, err)
-				require.ErrorContains(t, err, tc.expectError)
+				assert.Error(t, err)
+				assert.ErrorContains(t, err, tc.expectError)
 				return
 			}
 			require.NoError(t, err)
@@ -142,12 +156,13 @@ func TestCreateSignedSTSIdentityRequest(t *testing.T) {
 			// parameters were correctly included by the AWS SDK.
 			httpReq, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(req)))
 			require.NoError(t, err)
-			require.Equal(t, tc.expectEndpoint, httpReq.Host)
+			assert.Equal(t, tc.expectEndpoint, httpReq.Host)
 			authHeader := httpReq.Header.Get(aws.AuthorizationHeader)
 			sigV4, err := aws.ParseSigV4(authHeader)
 			require.NoError(t, err)
-			require.Contains(t, sigV4.SignedHeaders, "x-teleport-challenge")
-			require.Equal(t, challenge, httpReq.Header.Get("x-teleport-challenge"))
+			assert.Contains(t, sigV4.SignedHeaders, "x-teleport-challenge")
+			assert.Equal(t, challenge, httpReq.Header.Get("x-teleport-challenge"))
+			assert.Equal(t, tc.expectSignatureRegion, sigV4.Region, "signature region did not match expected")
 		})
 	}
 }