From 7cfe73b0e3e5b7ade4c210207c7cb6ea3f42172b Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 4 Jan 2024 13:41:35 -0500 Subject: [PATCH 001/145] Add DisableExpressSessionAuth config --- .../codegen/customization/s3/ClientConfigIntegration.kt | 9 +++++++++ .../aws/sdk/kotlin/codegen/endpoints/AwsBuiltins.kt | 1 + .../kotlin/codegen/endpoints/BindAwsEndpointBuiltins.kt | 1 + .../sdk/kotlin/services/s3/internal/FinalizeS3Config.kt | 4 ++++ 4 files changed, 15 insertions(+) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index 5926c47e98e..5e82290bf19 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -62,6 +62,14 @@ class ClientConfigIntegration : KotlinIntegration { Flag to disable [S3 multi-region access points](https://docs.aws.amazon.com/AmazonS3/latest/userguide/MultiRegionAccessPoints.html). """.trimIndent() } + + val DisableExpressSessionAuth: ConfigProperty = ConfigProperty { + name = "disableExpressSessionAuth" + useSymbolWithNullableBuilder(KotlinTypes.Boolean, "false") + documentation = """ + Flag to disable S3 Express One Zone's bucket-level session authentication method. + """.trimIndent() + } } override fun preprocessModel(model: Model, settings: KotlinSettings): Model { @@ -87,6 +95,7 @@ class ClientConfigIntegration : KotlinIntegration { ForcePathStyleProp, UseArnRegionProp, DisableMrapProp, + DisableExpressSessionAuth ) override val sectionWriters: List diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/AwsBuiltins.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/AwsBuiltins.kt index 6d90b7b8f14..c6e1924ea47 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/AwsBuiltins.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/AwsBuiltins.kt @@ -11,6 +11,7 @@ object AwsBuiltins { const val S3_ACCELERATE = "AWS::S3::Accelerate" const val S3_FORCE_PATH_STYLE = "AWS::S3::ForcePathStyle" const val S3_DISABLE_MRAP = "AWS::S3::DisableMultiRegionAccessPoints" + const val S3_DISABLE_EXPRESS_SESSION_AUTH = "AWS::S3::DisableS3ExpressSessionAuth" const val S3_USE_ARN_REGION = "AWS::S3::UseArnRegion" const val S3_CONTROL_USE_ARN_REGION = "AWS::S3Control::UseArnRegion" const val S3_USE_GLOBAL_ENDPOINT = "AWS::S3::UseGlobalEndpoint" diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/BindAwsEndpointBuiltins.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/BindAwsEndpointBuiltins.kt index 6643d3f8e77..46dea048c53 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/BindAwsEndpointBuiltins.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/BindAwsEndpointBuiltins.kt @@ -90,6 +90,7 @@ fun renderBindAwsBuiltins(ctx: ProtocolGenerator.GenerationContext, writer: Kotl AwsBuiltins.S3_ACCELERATE -> renderBasicConfigBinding(writer, it, S3ClientConfigIntegration.EnableAccelerateProp.propertyName) AwsBuiltins.S3_FORCE_PATH_STYLE -> renderBasicConfigBinding(writer, it, S3ClientConfigIntegration.ForcePathStyleProp.propertyName) AwsBuiltins.S3_DISABLE_MRAP -> renderBasicConfigBinding(writer, it, S3ClientConfigIntegration.DisableMrapProp.propertyName) + AwsBuiltins.S3_DISABLE_EXPRESS_SESSION_AUTH -> renderBasicConfigBinding(writer, it, S3ClientConfigIntegration.DisableExpressSessionAuth.propertyName) AwsBuiltins.S3_USE_ARN_REGION -> renderBasicConfigBinding(writer, it, S3ClientConfigIntegration.UseArnRegionProp.propertyName) AwsBuiltins.S3_CONTROL_USE_ARN_REGION -> renderBasicConfigBinding(writer, it, S3ControlClientConfigIntegration.UseArnRegionProp.propertyName) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt index cf75cc2bafd..be642828c91 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt @@ -20,6 +20,7 @@ internal suspend fun finalizeS3Config( val activeProfile = sharedConfig.get().activeProfile builder.config.useArnRegion = builder.config.useArnRegion ?: S3Setting.UseArnRegion.resolve(provider) ?: activeProfile.useArnRegion builder.config.disableMrap = builder.config.disableMrap ?: S3Setting.DisableMultiRegionAccessPoints.resolve(provider) ?: activeProfile.disableMrap + builder.config.disableExpressSessionAuth = builder.config.disableExpressSessionAuth ?: S3Setting.DisableExpressSessionAuth.resolve(provider) ?: activeProfile.disableExpressSessionAuth } private val AwsProfile.useArnRegion: Boolean? @@ -27,3 +28,6 @@ private val AwsProfile.useArnRegion: Boolean? private val AwsProfile.disableMrap: Boolean? get() = getBooleanOrNull("s3_disable_multiregion_access_points") + +private val AwsProfile.disableExpressSessionAuth: Boolean? + get() = getBooleanOrNull("s3_disable_express_session_auth") \ No newline at end of file From 4c3fc22154b37f2aef3d929105c5f1a781f11f5f Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 4 Jan 2024 14:32:11 -0500 Subject: [PATCH 002/145] Add DisableExpressSessionAuth config --- .../src/aws/sdk/kotlin/services/s3/internal/S3Setting.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3Setting.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3Setting.kt index bd90cd01200..34746e677b0 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3Setting.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3Setting.kt @@ -21,4 +21,9 @@ internal object S3Setting { * See [Amazon S3 Multi-Region Access Points](https://docs.aws.amazon.com/sdkref/latest/guide/feature-s3-mrap.html) */ public val DisableMultiRegionAccessPoints: EnvironmentSetting = boolEnvSetting("aws.s3DisableMultiRegionAccessPoints", "AWS_S3_DISABLE_MULTIREGION_ACCESS_POINTS") + + /** + * Configure whether requests made to S3 Express One Zone should use bucket-level session authentication or the default S3 authentication method. + */ + public val DisableExpressSessionAuth: EnvironmentSetting = boolEnvSetting("aws.s3DisableExpressSessionAuth", "AWS_S3_DISABLE_EXPRESS_SESSION_AUTH") } From 551acaf331abc8056015ce96435675ed8a7dc8c3 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 4 Jan 2024 14:32:41 -0500 Subject: [PATCH 003/145] .api --- aws-runtime/aws-config/api/aws-config.api | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index 7d947809b08..c5e4087efa0 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -115,6 +115,12 @@ public final class aws/sdk/kotlin/runtime/auth/credentials/ProviderConfiguration public synthetic fun (Ljava/lang/String;Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V } +public final class aws/sdk/kotlin/runtime/auth/credentials/S3ExpressCredentialsProvider : aws/smithy/kotlin/runtime/auth/awscredentials/CredentialsProvider { + public fun (Laws/smithy/kotlin/runtime/auth/awscredentials/CredentialsProvider;)V + public final fun getBootstrapCredentialsProvider ()Laws/smithy/kotlin/runtime/auth/awscredentials/CredentialsProvider; + public fun resolve (Laws/smithy/kotlin/runtime/collections/Attributes;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; +} + public final class aws/sdk/kotlin/runtime/auth/credentials/SsoCredentialsProvider : aws/smithy/kotlin/runtime/auth/awscredentials/CredentialsProvider { public fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/http/engine/HttpClientEngine;Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/time/Clock;)V public synthetic fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/http/engine/HttpClientEngine;Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/time/Clock;ILkotlin/jvm/internal/DefaultConstructorMarker;)V From d63fa12cce6d5167477e69b1af58709f6dfaa35d Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 11 Jan 2024 09:57:00 -0500 Subject: [PATCH 004/145] Latest commit --- aws-runtime/aws-config/api/aws-config.api | 32 ++++- .../runtime/auth/S3ExpressAttributes.kt | 13 ++ .../runtime/auth/SigV4S3ExpressAuthScheme.kt | 91 ++++++++++++ .../credentials/internal/CredentialsExt.kt | 2 +- .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 7 + .../s3/ClientConfigIntegration.kt | 18 ++- ...ConfigureS3ExpressAuthSchemeIntegration.kt | 85 +++++++++++ .../s3/express/S3ExpressCRC32Checksum.kt | 0 .../SigV4S3ExpressAuthSchemeIntegration.kt | 134 ++++++++++++++++++ .../s3/express/SigV4S3ExpressTrait.kt | 24 ++++ ...tlin.codegen.integration.KotlinIntegration | 4 +- gradle/libs.versions.toml | 4 +- .../internal/S3ExpressCredentialsProvider.kt | 35 +++++ .../s3/internal/S3ExpressIdentityCache.kt | 84 +++++++++++ 14 files changed, 522 insertions(+), 11 deletions(-) create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/ConfigureS3ExpressAuthSchemeIntegration.kt create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressCRC32Checksum.kt create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressTrait.kt create mode 100644 services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt create mode 100644 services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressIdentityCache.kt diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index c5e4087efa0..36bed72ad2d 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -1,3 +1,24 @@ +public final class aws/sdk/kotlin/runtime/auth/S3ExpressAttributes { + public static final field INSTANCE Laws/sdk/kotlin/runtime/auth/S3ExpressAttributes; + public final fun getBucket ()Laws/smithy/kotlin/runtime/collections/AttributeKey; + public final fun getClient ()Laws/smithy/kotlin/runtime/collections/AttributeKey; +} + +public final class aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme : aws/smithy/kotlin/runtime/http/auth/AuthScheme { + public fun (Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner;Ljava/lang/String;)V + public synthetic fun (Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner$Config;)V + public fun getSchemeId-DepwgT4 ()Ljava/lang/String; + public fun getSigner ()Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner; + public synthetic fun getSigner ()Laws/smithy/kotlin/runtime/http/auth/HttpSigner; + public fun identityProvider (Laws/smithy/kotlin/runtime/identity/IdentityProviderConfig;)Laws/smithy/kotlin/runtime/identity/IdentityProvider; +} + +public final class aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthSchemeKt { + public static final fun sigV4S3Express (ZLjava/lang/String;Ljava/lang/String;Ljava/lang/Boolean;Ljava/lang/Boolean;)Laws/smithy/kotlin/runtime/auth/AuthOption; + public static synthetic fun sigV4S3Express$default (ZLjava/lang/String;Ljava/lang/String;Ljava/lang/Boolean;Ljava/lang/Boolean;ILjava/lang/Object;)Laws/smithy/kotlin/runtime/auth/AuthOption; +} + public final class aws/sdk/kotlin/runtime/auth/credentials/AssumeRoleParameters { public synthetic fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;JLjava/util/List;Ljava/lang/String;Ljava/util/Map;Ljava/util/List;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;JLjava/util/List;Ljava/lang/String;Ljava/util/Map;Ljava/util/List;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/internal/DefaultConstructorMarker;)V @@ -115,12 +136,6 @@ public final class aws/sdk/kotlin/runtime/auth/credentials/ProviderConfiguration public synthetic fun (Ljava/lang/String;Ljava/lang/Throwable;ILkotlin/jvm/internal/DefaultConstructorMarker;)V } -public final class aws/sdk/kotlin/runtime/auth/credentials/S3ExpressCredentialsProvider : aws/smithy/kotlin/runtime/auth/awscredentials/CredentialsProvider { - public fun (Laws/smithy/kotlin/runtime/auth/awscredentials/CredentialsProvider;)V - public final fun getBootstrapCredentialsProvider ()Laws/smithy/kotlin/runtime/auth/awscredentials/CredentialsProvider; - public fun resolve (Laws/smithy/kotlin/runtime/collections/Attributes;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; -} - public final class aws/sdk/kotlin/runtime/auth/credentials/SsoCredentialsProvider : aws/smithy/kotlin/runtime/auth/awscredentials/CredentialsProvider { public fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/http/engine/HttpClientEngine;Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/time/Clock;)V public synthetic fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/http/engine/HttpClientEngine;Laws/smithy/kotlin/runtime/util/PlatformProvider;Laws/smithy/kotlin/runtime/time/Clock;ILkotlin/jvm/internal/DefaultConstructorMarker;)V @@ -209,6 +224,11 @@ public final class aws/sdk/kotlin/runtime/auth/credentials/SystemPropertyCredent public fun resolve (Laws/smithy/kotlin/runtime/collections/Attributes;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; } +public final class aws/sdk/kotlin/runtime/auth/credentials/internal/CredentialsExtKt { + public static final fun credentials (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/time/Instant;Ljava/lang/String;Ljava/lang/String;)Laws/smithy/kotlin/runtime/auth/awscredentials/Credentials; + public static synthetic fun credentials$default (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/time/Instant;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Laws/smithy/kotlin/runtime/auth/awscredentials/Credentials; +} + public final class aws/sdk/kotlin/runtime/auth/credentials/internal/ManagedBearerTokenProviderKt { public static final fun manage (Laws/smithy/kotlin/runtime/http/auth/CloseableBearerTokenProvider;)Laws/smithy/kotlin/runtime/http/auth/BearerTokenProvider; } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt new file mode 100644 index 00000000000..71a97c24a53 --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt @@ -0,0 +1,13 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.runtime.auth + +import aws.smithy.kotlin.runtime.client.SdkClient +import aws.smithy.kotlin.runtime.collections.AttributeKey + +public object S3ExpressAttributes { + public val Bucket: AttributeKey = AttributeKey("aws.sdk.kotlin#S3ExpressBucket") + public val Client: AttributeKey = AttributeKey("aws.sdk.kotlin#S3ExpressClient") +} \ No newline at end of file diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt new file mode 100644 index 00000000000..866272ce92d --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt @@ -0,0 +1,91 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.runtime.auth + +import aws.smithy.kotlin.runtime.InternalApi +import aws.smithy.kotlin.runtime.auth.AuthOption +import aws.smithy.kotlin.runtime.auth.AuthSchemeId +import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigner +import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAlgorithm +import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAttributes +import aws.smithy.kotlin.runtime.auth.awssigning.HashSpecification +import aws.smithy.kotlin.runtime.collections.AttributeKey +import aws.smithy.kotlin.runtime.collections.MutableAttributes +import aws.smithy.kotlin.runtime.collections.emptyAttributes +import aws.smithy.kotlin.runtime.collections.mutableAttributes +import aws.smithy.kotlin.runtime.http.auth.AuthScheme +import aws.smithy.kotlin.runtime.http.auth.AwsHttpSigner + +/** + * HTTP auth scheme for S3 Express One Zone authentication + */ +public class SigV4S3ExpressAuthScheme( + config: AwsHttpSigner.Config, +) : AuthScheme { + public constructor(awsSigner: AwsSigner, serviceName: String? = null) : this( + AwsHttpSigner.Config().apply { + signer = awsSigner + service = serviceName + algorithm = AwsSigningAlgorithm.SIGV4_S3EXPRESS + }, + ) + + override val schemeId: AuthSchemeId = AuthSchemeId.AwsSigV4S3Express + override val signer: AwsHttpSigner = AwsHttpSigner(config) +} + +/** + * Create a new [AuthOption] for the [SigV4S3ExpressAuthScheme] + * @param unsignedPayload set the signing attribute to indicate the signer should use unsigned payload. + * @param serviceName override the service name to sign for + * @param signingRegion override the signing region to sign for + * @param disableDoubleUriEncode disable double URI encoding + * @param normalizeUriPath flag indicating if the URI path should be normalized when forming the canonical request + * @return auth scheme option representing the [SigV4S3ExpressAuthScheme] + */ +@InternalApi +public fun sigV4S3Express( + unsignedPayload: Boolean = false, + serviceName: String? = null, + signingRegion: String? = null, + disableDoubleUriEncode: Boolean? = null, + normalizeUriPath: Boolean? = null, +): AuthOption { + val attrs = if (unsignedPayload || serviceName != null || signingRegion != null || disableDoubleUriEncode != null || normalizeUriPath != null) { + val mutAttrs = mutableAttributes() + mutAttrs.setNotBlank(AwsSigningAttributes.SigningRegion, signingRegion) + setCommonSigV4Attrs(mutAttrs, unsignedPayload, serviceName, disableDoubleUriEncode, normalizeUriPath) + mutAttrs + } else { + emptyAttributes() + } + return AuthOption(AuthSchemeId.AwsSigV4S3Express, attrs) +} + +// FIXME Copied from SigV4AuthScheme.kt +internal fun MutableAttributes.setNotBlank(key: AttributeKey, value: String?) { + if (!value.isNullOrBlank()) set(key, value) +} + +internal fun setCommonSigV4Attrs( + attrs: MutableAttributes, + unsignedPayload: Boolean = false, + serviceName: String? = null, + disableDoubleUriEncode: Boolean? = null, + normalizeUriPath: Boolean? = null, +) { + if (unsignedPayload) { + attrs[AwsSigningAttributes.HashSpecification] = HashSpecification.UnsignedPayload + } + attrs.setNotBlank(AwsSigningAttributes.SigningService, serviceName) + if (disableDoubleUriEncode != null) { + attrs[AwsSigningAttributes.UseDoubleUriEncode] = !disableDoubleUriEncode + } + + if (normalizeUriPath != null) { + attrs[AwsSigningAttributes.NormalizeUriPath] = normalizeUriPath + } +} diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/internal/CredentialsExt.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/internal/CredentialsExt.kt index 3f40a0e584d..b98822f457a 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/internal/CredentialsExt.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/internal/CredentialsExt.kt @@ -12,7 +12,7 @@ import aws.smithy.kotlin.runtime.collections.setIfValueNotNull import aws.smithy.kotlin.runtime.identity.IdentityAttributes import aws.smithy.kotlin.runtime.time.Instant -internal fun credentials( +public fun credentials( // FIXME can we make this public? if not, how should I create [Credentials] from within the s3 package? accessKeyId: String, secretAccessKey: String, sessionToken: String? = null, diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index 438eef7a95e..58820b17417 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -52,8 +52,15 @@ object AwsRuntimeTypes { val DefaultChainCredentialsProvider = symbol("DefaultChainCredentialsProvider") val DefaultChainBearerTokenProvider = symbol("DefaultChainBearerTokenProvider") val StaticCredentialsProvider = symbol("StaticCredentialsProvider") + val S3ExpressCredentialsProvider = symbol("S3ExpressCredentialsProvider") val manage = symbol("manage", "auth.credentials.internal", isExtension = true) } + + object Auth : RuntimeTypePackage(AwsKotlinDependency.AWS_CONFIG, "auth") { + val SigV4S3ExpressAuthScheme = symbol("SigV4S3ExpressAuthScheme") + val sigV4S3Express = symbol("sigV4S3Express") + val S3ExpressAttributes = symbol("S3ExpressAttributes") + } } object Http : RuntimeTypePackage(AwsKotlinDependency.AWS_HTTP) { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index 5e82290bf19..9e7f4d97972 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -4,6 +4,7 @@ */ package aws.sdk.kotlin.codegen.customization.s3 +import aws.sdk.kotlin.codegen.AwsRuntimeTypes import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.CodegenContext import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter @@ -14,6 +15,7 @@ import software.amazon.smithy.kotlin.codegen.model.buildSymbol import software.amazon.smithy.kotlin.codegen.model.expectShape import software.amazon.smithy.kotlin.codegen.rendering.ServiceClientGenerator import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.ServiceShape import software.amazon.smithy.model.transform.ModelTransformer @@ -70,6 +72,19 @@ class ClientConfigIntegration : KotlinIntegration { Flag to disable S3 Express One Zone's bucket-level session authentication method. """.trimIndent() } + + val S3ExpressCredentialsProvider: ConfigProperty = ConfigProperty { + name = "s3ExpressCredentialsProvider" + propertyType = ConfigPropertyType.RequiredWithDefault("S3ExpressCredentialsProvider(config.credentialsProvider)") + useSymbolWithNullableBuilder(buildSymbol { + name = "S3ExpressCredentialsProvider" + nullable = false + namespace = "aws.sdk.kotlin.services.s3" + }) + documentation = """ + Credentials provider to be used for making requests to S3 Express. + """.trimIndent() + } } override fun preprocessModel(model: Model, settings: KotlinSettings): Model { @@ -95,7 +110,8 @@ class ClientConfigIntegration : KotlinIntegration { ForcePathStyleProp, UseArnRegionProp, DisableMrapProp, - DisableExpressSessionAuth + DisableExpressSessionAuth, + S3ExpressCredentialsProvider, ) override val sectionWriters: List diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/ConfigureS3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/ConfigureS3ExpressAuthSchemeIntegration.kt new file mode 100644 index 00000000000..1d3ac2a0d48 --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/ConfigureS3ExpressAuthSchemeIntegration.kt @@ -0,0 +1,85 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.codegen.customization.s3.express + +import aws.sdk.kotlin.codegen.AwsRuntimeTypes +import aws.sdk.kotlin.codegen.customization.s3.ClientConfigIntegration +import aws.sdk.kotlin.codegen.customization.s3.isS3 +import software.amazon.smithy.aws.traits.HttpChecksumTrait +import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.KotlinWriter +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Auth.Identity.AuthSchemeId +import software.amazon.smithy.kotlin.codegen.core.defaultName +import software.amazon.smithy.kotlin.codegen.core.withBlock +import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter +import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding +import software.amazon.smithy.kotlin.codegen.model.expectShape +import software.amazon.smithy.kotlin.codegen.model.getTrait +import software.amazon.smithy.kotlin.codegen.rendering.auth.AuthSchemeProviderGenerator +import software.amazon.smithy.kotlin.codegen.rendering.auth.IdentityProviderConfigGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpProtocolClientGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.kotlin.codegen.utils.getOrNull +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.model.shapes.StructureShape + +class ConfigureS3ExpressAuthSchemeIntegration : KotlinIntegration { + override fun enabledForService(model: Model, settings: KotlinSettings) = model.expectShape(settings.service).isS3 + + override val sectionWriters: List + get() = listOf( + SectionWriterBinding(HttpProtocolClientGenerator.ConfigureAuthSchemes, configureS3ExpressAuthSchemeWriter), + SectionWriterBinding(AuthSchemeProviderGenerator.ServiceDefaults, setServiceDefaultAuthOptionWriter), + SectionWriterBinding(IdentityProviderConfigGenerator.ConfigureIdentityProviderForAuthScheme, configureIdentityProviderForAuthSchemeWriter), + ) + + private val configureS3ExpressAuthSchemeWriter = AppendingSectionWriter { writer -> + writer.withBlock("getOrPut(#T) {", "}", SigV4S3ExpressAuthSchemeHandler().authSchemeIdSymbol) { + writer.write("#T(#T, #S)", AwsRuntimeTypes.Config.Auth.SigV4S3ExpressAuthScheme, RuntimeTypes.Auth.Signing.AwsSigningStandard.DefaultAwsSigner, "s3") + } + } + + private val setServiceDefaultAuthOptionWriter = AppendingSectionWriter { writer -> + writer.write("#T(),", AwsRuntimeTypes.Config.Auth.sigV4S3Express) + } + + private val configureIdentityProviderForAuthSchemeWriter = AppendingSectionWriter { writer -> + writer.write("#S -> config.#L", "aws.auth#sigv4s3express", ClientConfigIntegration.S3ExpressCredentialsProvider.propertyName) + } + + override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = + resolved + AddClientToExecutionContext + AddBucketToExecutionContext + + private val AddClientToExecutionContext = object : ProtocolMiddleware { + override val name: String = "AddClientToExecutionContext" + + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = ctx.model.expectShape(ctx.settings.service).isS3 + + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val attributesSymbol = AwsRuntimeTypes.Config.Auth.S3ExpressAttributes + writer.write("op.context[#T.Client] = this", attributesSymbol) + } + } + + private val AddBucketToExecutionContext = object : ProtocolMiddleware { + override val name: String = "AddBucketToExecutionContext" + + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + ctx.model.expectShape(op.input.get()) + .members() + .any { it.memberName == "Bucket" } + + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val attributesSymbol = AwsRuntimeTypes.Config.Auth.S3ExpressAttributes + writer.write("input.bucket?.let { op.context[#T.Bucket] = it }", attributesSymbol) + } + } + +} \ No newline at end of file diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressCRC32Checksum.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressCRC32Checksum.kt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt new file mode 100644 index 00000000000..2fddcca6dd0 --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -0,0 +1,134 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.codegen.customization.s3.express + +import aws.sdk.kotlin.codegen.AwsRuntimeTypes +import aws.sdk.kotlin.codegen.customization.s3.ClientConfigIntegration +import aws.sdk.kotlin.codegen.customization.s3.isS3 +import software.amazon.smithy.aws.traits.auth.UnsignedPayloadTrait +import software.amazon.smithy.codegen.core.Symbol +import software.amazon.smithy.codegen.core.SymbolReference +import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.* +import software.amazon.smithy.kotlin.codegen.integration.AuthSchemeHandler +import software.amazon.smithy.kotlin.codegen.model.buildSymbol +import software.amazon.smithy.kotlin.codegen.model.expectShape +import software.amazon.smithy.kotlin.codegen.model.hasTrait +import software.amazon.smithy.kotlin.codegen.model.knowledge.AwsSignatureVersion4 +import software.amazon.smithy.kotlin.codegen.rendering.auth.SigV4AuthSchemeIntegration +import software.amazon.smithy.kotlin.codegen.rendering.endpoints.EndpointCustomization +import software.amazon.smithy.kotlin.codegen.rendering.endpoints.EndpointPropertyRenderer +import software.amazon.smithy.kotlin.codegen.rendering.endpoints.ExpressionRenderer +import software.amazon.smithy.kotlin.codegen.rendering.protocol.* +import software.amazon.smithy.kotlin.codegen.utils.getOrNull +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.node.Node +import software.amazon.smithy.model.node.ObjectNode +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.model.shapes.ShapeId +import software.amazon.smithy.rulesengine.language.syntax.expressions.Expression +import java.util.* + +/** + * Register support for the `sigv4-s3express` auth scheme. + */ +class SigV4S3ExpressAuthSchemeIntegration : SigV4AuthSchemeIntegration() { + override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = model.expectShape(settings.service).isS3 + + override fun customizeEndpointResolution(ctx: ProtocolGenerator.GenerationContext): EndpointCustomization = SigV4S3ExpressEndpointCustomization + + override fun authSchemes(ctx: ProtocolGenerator.GenerationContext): List = listOf(SigV4S3ExpressAuthSchemeHandler()) +} + +private object SigV4S3ExpressEndpointCustomization : EndpointCustomization { + override val propertyRenderers: Map = mapOf( + "authSchemes" to ::renderAuthSchemes, + ) +} + +open class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { + override val authSchemeId: ShapeId = SigV4S3ExpressTrait.ID + + override val authSchemeIdSymbol: Symbol = buildSymbol { + name = "AuthSchemeId.AwsSigV4S3Express" + val ref = RuntimeTypes.Auth.Identity.AuthSchemeId + objectRef = ref + namespace = ref.namespace + reference(ref, SymbolReference.ContextOption.USE) + } + + override fun identityProviderAdapterExpression(writer: KotlinWriter) { + writer.write("config.#T", ClientConfigIntegration.S3ExpressCredentialsProvider.propertyName) + } + + override fun authSchemeProviderInstantiateAuthOptionExpr( + ctx: ProtocolGenerator.GenerationContext, + op: OperationShape?, + writer: KotlinWriter, + ) { + val expr = if (op?.hasTrait() == true) { + "#T(unsignedPayload = true)" + } else { + "#T()" + } + writer.write(expr, AwsRuntimeTypes.Config.Auth.sigV4S3Express) + } + + override fun instantiateAuthSchemeExpr(ctx: ProtocolGenerator.GenerationContext, writer: KotlinWriter) { + val signingService = AwsSignatureVersion4.signingServiceName(ctx.service) + writer.write("#T(#T, #S)", AwsRuntimeTypes.Config.Auth.SigV4S3ExpressAuthScheme, RuntimeTypes.Auth.Signing.AwsSigningStandard.DefaultAwsSigner, signingService) + } +} + +private fun String.toAuthOptionFactoryFn(): Symbol? = + when (this) { + "sigv4-s3express" -> AwsRuntimeTypes.Config.Auth.sigV4S3Express + else -> null + } + +private fun renderAuthSchemes(writer: KotlinWriter, authSchemes: Expression, expressionRenderer: ExpressionRenderer) { + writer.writeInline("#T to ", RuntimeTypes.SmithyClient.Endpoints.SigningContextAttributeKey) + writer.withBlock("listOf(", ")") { + authSchemes.toNode().expectArrayNode().forEach { + val scheme = it.expectObjectNode() + val schemeName = scheme.expectStringMember("name").value + val authFactoryFn = schemeName.toAuthOptionFactoryFn() ?: return@forEach + + withBlock("#T(", "),", authFactoryFn) { + // we delegate back to the expression visitor for each of these fields because it's possible to + // encounter template strings throughout + + writeInline("serviceName = ") + renderOrElse(expressionRenderer, scheme.getStringMember("signingName"), "null") + + writeInline("disableDoubleUriEncode = ") + renderOrElse(expressionRenderer, scheme.getBooleanMember("disableDoubleEncoding"), "false") + + when (schemeName) { + "sigv4-s3express" -> renderSigV4ExpressFields(writer, scheme, expressionRenderer) + } + } + } + } +} + +private fun KotlinWriter.renderOrElse( + expressionRenderer: ExpressionRenderer, + optionalNode: Optional, + whenNullValue: String, +) { + val nullableNode = optionalNode.getOrNull() + when (nullableNode) { + null -> writeInline(whenNullValue) + else -> expressionRenderer.renderExpression(Expression.fromNode(nullableNode)) + } + write(",") +} + +private fun renderSigV4ExpressFields(writer: KotlinWriter, scheme: ObjectNode, expressionRenderer: ExpressionRenderer) { + writer.writeInline("signingRegion = ") + writer.renderOrElse(expressionRenderer, scheme.getStringMember("signingRegion"), "null") +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressTrait.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressTrait.kt new file mode 100644 index 00000000000..2b7e11de406 --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressTrait.kt @@ -0,0 +1,24 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.codegen.customization.s3.express + +import software.amazon.smithy.model.node.Node +import software.amazon.smithy.model.shapes.ShapeId +import software.amazon.smithy.model.traits.AbstractTrait + +/** + * Synthetic trait representing the `sigv4-s3express` auth scheme. + */ +public class SigV4S3ExpressTrait : AbstractTrait(ID, Node.objectNode()) { + public companion object { + val ID = ShapeId.from("com.amazonaws.s3#sigv4express") + // FIXME What shape ID should be used for sigv4-s3express? It's not in Smithy... + // Go v2 uses this. https://github.com/aws/aws-sdk-go-v2/blob/4ba37053fff9055b216a303ab591f6fa5e4c80c1/service/s3/endpoint_auth_resolver.go#L29C1-L29C1 + } + + override fun createNode(): Node = Node.objectNode() + + override fun isSynthetic(): Boolean = true +} diff --git a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration index 9c1bcbd6748..9702bd5c869 100644 --- a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +++ b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration @@ -20,12 +20,12 @@ aws.sdk.kotlin.codegen.customization.s3.ClientConfigIntegration aws.sdk.kotlin.codegen.customization.s3.ContinueIntegration aws.sdk.kotlin.codegen.customization.s3.GetObjectResponseLengthValidationIntegration aws.sdk.kotlin.codegen.customization.s3.HttpPathFilter +aws.sdk.kotlin.codegen.customization.s3.S3ErrorWith200StatusIntegration aws.sdk.kotlin.codegen.customization.s3.TruncatablePaginationIntegration aws.sdk.kotlin.codegen.customization.s3.HostPrefixRequestRouteFilter aws.sdk.kotlin.codegen.customization.s3.UnwrappedXmlOutputIntegration aws.sdk.kotlin.codegen.customization.s3control.HostPrefixFilter aws.sdk.kotlin.codegen.customization.s3control.ClientConfigIntegration -aws.sdk.kotlin.codegen.customization.s3.S3ErrorWith200StatusIntegration aws.sdk.kotlin.codegen.customization.apigateway.ApiGatewayAddAcceptHeader aws.sdk.kotlin.codegen.customization.glacier.GlacierAddVersionHeader aws.sdk.kotlin.codegen.customization.glacier.GlacierAccountIdDefault @@ -36,3 +36,5 @@ aws.sdk.kotlin.codegen.customization.route53.TrimResourcePrefix aws.sdk.kotlin.codegen.customization.route53.ChangeResourceRecordSetsUnmarshallingIntegration aws.sdk.kotlin.codegen.customization.ec2.EC2MakePrimitivesOptional aws.sdk.kotlin.codegen.customization.RemoveDefaults +aws.sdk.kotlin.codegen.customization.s3.express.SigV4S3ExpressAuthSchemeIntegration +aws.sdk.kotlin.codegen.customization.s3.express.ConfigureS3ExpressAuthSchemeIntegration diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 54a2b215e36..daac61c8d86 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -9,8 +9,8 @@ coroutines-version = "1.7.3" atomicfu-version = "0.23.1" # smithy-kotlin codegen and runtime are versioned separately -smithy-kotlin-runtime-version = "1.0.7" -smithy-kotlin-codegen-version = "0.30.8" +smithy-kotlin-runtime-version = "1.0.8-SNAPSHOT" +smithy-kotlin-codegen-version = "0.30.9-SNAPSHOT" # codegen smithy-version = "1.42.0" diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt new file mode 100644 index 00000000000..f0cb0595e28 --- /dev/null +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.services.s3.internal + +import aws.sdk.kotlin.runtime.auth.S3ExpressAttributes +import aws.sdk.kotlin.runtime.auth.credentials.internal.S3ExpressCredentialsCache +import aws.sdk.kotlin.runtime.auth.credentials.internal.S3ExpressCredentialsCacheKey +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider +import aws.smithy.kotlin.runtime.collections.Attributes +import aws.smithy.kotlin.runtime.collections.get +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlin.coroutines.coroutineContext + +public class S3ExpressCredentialsProvider( + public val bootstrapCredentialsProvider: CredentialsProvider +) : CredentialsProvider { + private val credentialsCache = S3ExpressCredentialsCache() + + override suspend fun resolve(attributes: Attributes): Credentials { + val logger = coroutineContext.logger() + logger.debug { "received attributes: ${attributes.keys.joinToString { it.name }}"} + + val bucket: String = attributes[S3ExpressAttributes.Bucket] + val client = attributes[S3ExpressAttributes.Client] + + val key = S3ExpressCredentialsCacheKey(bucket, client, bootstrapCredentialsProvider.resolve(attributes)) + + logger.debug { "fetching credentials from cache with key $key"} + + return credentialsCache.get(key) + } +} diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressIdentityCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressIdentityCache.kt new file mode 100644 index 00000000000..c637cd5d3cf --- /dev/null +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressIdentityCache.kt @@ -0,0 +1,84 @@ +package aws.sdk.kotlin.runtime.auth.credentials.internal + +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CreateSessionRequest +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.client.SdkClient +import aws.smithy.kotlin.runtime.collections.LruCache +import aws.smithy.kotlin.runtime.time.Clock +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import aws.smithy.kotlin.runtime.util.ExpiringValue +import kotlin.coroutines.coroutineContext + +private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 + +public class S3ExpressCredentialsCache( + private val clock: Clock = Clock.System +) { + private val lru = LruCache>(DEFAULT_S3_EXPRESS_CACHE_SIZE) + + public suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials { + val logger = coroutineContext.logger() + + val cached = lru.get(key) + + if (cached == null) { + logger.debug { "could not find cache value for key $key. entries: ${lru.entries.joinToString { it.key.toString() }}" } + } else if (cached.expiresAt > clock.now()) { + logger.debug { "found expired cache value for key $key: ${cached.value}" } + } else { + logger.debug { "got cached value ${cached.value}" } + } + + if (cached == null || cached.expiresAt > clock.now()) { + val newCredentials = createSessionCredentials(key) + + lru.put(key, newCredentials) + logger.debug { "fetched new credentials ${newCredentials.value}. entries: ${lru.entries.joinToString { it.key.toString() }}" } + return newCredentials.value + } + + return cached.value + } + + private suspend fun createSessionCredentials(key: S3ExpressCredentialsCacheKey): ExpiringValue { + val logger = coroutineContext.logger() + + val credentials = (key.client as S3Client).createSession(CreateSessionRequest { + bucket = key.bucket + }).credentials!! + + return ExpiringValue( + credentials( + accessKeyId = credentials.accessKeyId, + secretAccessKey = credentials.secretAccessKey, + sessionToken = credentials.sessionToken, + expiration = credentials.expiration, + providerName = "S3ExpressCredentialsProvider" + ), + credentials.expiration + ).also { logger.debug { "got credentials ${it.value}" } } + } +} + +public class S3ExpressCredentialsCacheKey( + public val bucket: String, + public val client: SdkClient, + public val credentials: Credentials +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is S3ExpressCredentialsCacheKey) return false + if (bucket != other.bucket) return false + if (client != other.client) return false + if (credentials != other.credentials) return false + return true + } + + override fun hashCode(): Int { + var result = bucket.hashCode() ?: 0 + result = 31 * result + (client.hashCode() ?: 0) + result = 31 * result + (credentials.hashCode() ?: 0) + return result + } +} \ No newline at end of file From 14e34792704e3a7960f3d56bca98d949454b32be Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 11 Jan 2024 10:50:05 -0500 Subject: [PATCH 005/145] ktlint --- .../runtime/auth/S3ExpressAttributes.kt | 2 +- .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 1 - .../s3/ClientConfigIntegration.kt | 20 ++++-- ...ConfigureS3ExpressAuthSchemeIntegration.kt | 8 +-- .../s3/express/S3ExpressCRC32Checksum.kt | 5 ++ .../services/s3/internal/FinalizeS3Config.kt | 2 +- .../internal/S3ExpressCredentialsProvider.kt | 33 ++++++++-- .../s3/internal/S3ExpressIdentityCache.kt | 62 ++++--------------- 8 files changed, 63 insertions(+), 70 deletions(-) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt index 71a97c24a53..fc7756211de 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt @@ -10,4 +10,4 @@ import aws.smithy.kotlin.runtime.collections.AttributeKey public object S3ExpressAttributes { public val Bucket: AttributeKey = AttributeKey("aws.sdk.kotlin#S3ExpressBucket") public val Client: AttributeKey = AttributeKey("aws.sdk.kotlin#S3ExpressClient") -} \ No newline at end of file +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index 58820b17417..bb6584f3098 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -52,7 +52,6 @@ object AwsRuntimeTypes { val DefaultChainCredentialsProvider = symbol("DefaultChainCredentialsProvider") val DefaultChainBearerTokenProvider = symbol("DefaultChainBearerTokenProvider") val StaticCredentialsProvider = symbol("StaticCredentialsProvider") - val S3ExpressCredentialsProvider = symbol("S3ExpressCredentialsProvider") val manage = symbol("manage", "auth.credentials.internal", isExtension = true) } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index 9e7f4d97972..407e270baf1 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -4,7 +4,6 @@ */ package aws.sdk.kotlin.codegen.customization.s3 -import aws.sdk.kotlin.codegen.AwsRuntimeTypes import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.CodegenContext import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter @@ -75,15 +74,26 @@ class ClientConfigIntegration : KotlinIntegration { val S3ExpressCredentialsProvider: ConfigProperty = ConfigProperty { name = "s3ExpressCredentialsProvider" - propertyType = ConfigPropertyType.RequiredWithDefault("S3ExpressCredentialsProvider(config.credentialsProvider)") - useSymbolWithNullableBuilder(buildSymbol { + symbol = buildSymbol { name = "S3ExpressCredentialsProvider" nullable = false - namespace = "aws.sdk.kotlin.services.s3" - }) + namespace = "aws.sdk.kotlin.services.s3.internal" + } documentation = """ Credentials provider to be used for making requests to S3 Express. """.trimIndent() + + // FIXME Figure out why the default value is not being used properly. +// propertyType = ConfigPropertyType.RequiredWithDefault("S3ExpressCredentialsProvider(this.credentialsProvider)") + propertyType = ConfigPropertyType.Custom( + render = { _, writer -> + writer.write("public val $name: S3ExpressCredentialsProvider = builder.$name ?: S3ExpressCredentialsProvider(this.credentialsProvider)") + }, + renderBuilder = { prop, writer -> + prop.documentation?.let(writer::dokka) + writer.write("public var $name: S3ExpressCredentialsProvider? = null") + }, + ) } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/ConfigureS3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/ConfigureS3ExpressAuthSchemeIntegration.kt index 1d3ac2a0d48..7e34d3d55c1 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/ConfigureS3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/ConfigureS3ExpressAuthSchemeIntegration.kt @@ -7,24 +7,19 @@ package aws.sdk.kotlin.codegen.customization.s3.express import aws.sdk.kotlin.codegen.AwsRuntimeTypes import aws.sdk.kotlin.codegen.customization.s3.ClientConfigIntegration import aws.sdk.kotlin.codegen.customization.s3.isS3 -import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.KotlinWriter import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes.Auth.Identity.AuthSchemeId -import software.amazon.smithy.kotlin.codegen.core.defaultName import software.amazon.smithy.kotlin.codegen.core.withBlock import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding import software.amazon.smithy.kotlin.codegen.model.expectShape -import software.amazon.smithy.kotlin.codegen.model.getTrait import software.amazon.smithy.kotlin.codegen.rendering.auth.AuthSchemeProviderGenerator import software.amazon.smithy.kotlin.codegen.rendering.auth.IdentityProviderConfigGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpProtocolClientGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware -import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ServiceShape @@ -81,5 +76,4 @@ class ConfigureS3ExpressAuthSchemeIntegration : KotlinIntegration { writer.write("input.bucket?.let { op.context[#T.Bucket] = it }", attributesSymbol) } } - -} \ No newline at end of file +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressCRC32Checksum.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressCRC32Checksum.kt index e69de29bb2d..d858e0d42ab 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressCRC32Checksum.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressCRC32Checksum.kt @@ -0,0 +1,5 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt index be642828c91..3e6a03fb594 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt @@ -30,4 +30,4 @@ private val AwsProfile.disableMrap: Boolean? get() = getBooleanOrNull("s3_disable_multiregion_access_points") private val AwsProfile.disableExpressSessionAuth: Boolean? - get() = getBooleanOrNull("s3_disable_express_session_auth") \ No newline at end of file + get() = getBooleanOrNull("s3_disable_express_session_auth") diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt index f0cb0595e28..5ad4b4b088a 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt @@ -7,29 +7,50 @@ package aws.sdk.kotlin.services.s3.internal import aws.sdk.kotlin.runtime.auth.S3ExpressAttributes import aws.sdk.kotlin.runtime.auth.credentials.internal.S3ExpressCredentialsCache import aws.sdk.kotlin.runtime.auth.credentials.internal.S3ExpressCredentialsCacheKey +import aws.sdk.kotlin.runtime.auth.credentials.internal.credentials +import aws.sdk.kotlin.services.s3.* +import aws.sdk.kotlin.services.s3.model.CreateSessionRequest import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.telemetry.logging.logger +import aws.smithy.kotlin.runtime.util.ExpiringValue import kotlin.coroutines.coroutineContext public class S3ExpressCredentialsProvider( - public val bootstrapCredentialsProvider: CredentialsProvider + public val bootstrapCredentialsProvider: CredentialsProvider, ) : CredentialsProvider { private val credentialsCache = S3ExpressCredentialsCache() override suspend fun resolve(attributes: Attributes): Credentials { - val logger = coroutineContext.logger() - logger.debug { "received attributes: ${attributes.keys.joinToString { it.name }}"} - val bucket: String = attributes[S3ExpressAttributes.Bucket] val client = attributes[S3ExpressAttributes.Client] val key = S3ExpressCredentialsCacheKey(bucket, client, bootstrapCredentialsProvider.resolve(attributes)) - logger.debug { "fetching credentials from cache with key $key"} - return credentialsCache.get(key) + ?: (createSessionCredentials(key).also { credentialsCache.put(key, it) }).value + } + + private suspend fun createSessionCredentials(key: S3ExpressCredentialsCacheKey): ExpiringValue { + val logger = coroutineContext.logger() + + val credentials = (key.client as S3Client).createSession( + CreateSessionRequest { + bucket = key.bucket + }, + ).credentials!! + + return ExpiringValue( + credentials( + accessKeyId = credentials.accessKeyId, + secretAccessKey = credentials.secretAccessKey, + sessionToken = credentials.sessionToken, + expiration = credentials.expiration, + providerName = "S3ExpressCredentialsProvider", + ), + credentials.expiration, + ).also { logger.debug { "got credentials ${it.value}" } } } } diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressIdentityCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressIdentityCache.kt index c637cd5d3cf..1df09e01093 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressIdentityCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressIdentityCache.kt @@ -1,70 +1,34 @@ -package aws.sdk.kotlin.runtime.auth.credentials.internal +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.services.s3.internal -import aws.sdk.kotlin.services.s3.S3Client -import aws.sdk.kotlin.services.s3.model.CreateSessionRequest import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.collections.LruCache import aws.smithy.kotlin.runtime.time.Clock -import aws.smithy.kotlin.runtime.telemetry.logging.logger import aws.smithy.kotlin.runtime.util.ExpiringValue -import kotlin.coroutines.coroutineContext private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 public class S3ExpressCredentialsCache( - private val clock: Clock = Clock.System + private val clock: Clock = Clock.System, ) { private val lru = LruCache>(DEFAULT_S3_EXPRESS_CACHE_SIZE) - public suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials { - val logger = coroutineContext.logger() + public suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials? = ( + lru.get(key) + ?.takeIf { it.expiresAt > clock.now() } + )?.value - val cached = lru.get(key) - - if (cached == null) { - logger.debug { "could not find cache value for key $key. entries: ${lru.entries.joinToString { it.key.toString() }}" } - } else if (cached.expiresAt > clock.now()) { - logger.debug { "found expired cache value for key $key: ${cached.value}" } - } else { - logger.debug { "got cached value ${cached.value}" } - } - - if (cached == null || cached.expiresAt > clock.now()) { - val newCredentials = createSessionCredentials(key) - - lru.put(key, newCredentials) - logger.debug { "fetched new credentials ${newCredentials.value}. entries: ${lru.entries.joinToString { it.key.toString() }}" } - return newCredentials.value - } - - return cached.value - } - - private suspend fun createSessionCredentials(key: S3ExpressCredentialsCacheKey): ExpiringValue { - val logger = coroutineContext.logger() - - val credentials = (key.client as S3Client).createSession(CreateSessionRequest { - bucket = key.bucket - }).credentials!! - - return ExpiringValue( - credentials( - accessKeyId = credentials.accessKeyId, - secretAccessKey = credentials.secretAccessKey, - sessionToken = credentials.sessionToken, - expiration = credentials.expiration, - providerName = "S3ExpressCredentialsProvider" - ), - credentials.expiration - ).also { logger.debug { "got credentials ${it.value}" } } - } + public suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue): Unit = lru.put(key, value) } public class S3ExpressCredentialsCacheKey( public val bucket: String, public val client: SdkClient, - public val credentials: Credentials + public val credentials: Credentials, ) { override fun equals(other: Any?): Boolean { if (this === other) return true @@ -81,4 +45,4 @@ public class S3ExpressCredentialsCacheKey( result = 31 * result + (credentials.hashCode() ?: 0) return result } -} \ No newline at end of file +} From 73d73d53b2865ad68f44185bfaab5808334a6e13 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 11 Jan 2024 11:47:38 -0500 Subject: [PATCH 006/145] latest commit --- .../codegen/customization/s3/ClientConfigIntegration.kt | 2 +- ...sAuthSchemeIntegration.kt => S3ExpressIntegration.kt} | 9 ++++++++- ...n.smithy.kotlin.codegen.integration.KotlinIntegration | 2 +- .../sdk/kotlin/services/s3/internal/FinalizeS3Config.kt | 4 ++-- ...ressIdentityCache.kt => S3ExpressCredentialsCache.kt} | 0 .../services/s3/internal/S3ExpressCredentialsProvider.kt | 2 -- .../src/aws/sdk/kotlin/services/s3/internal/S3Setting.kt | 2 +- 7 files changed, 13 insertions(+), 8 deletions(-) rename codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/{ConfigureS3ExpressAuthSchemeIntegration.kt => S3ExpressIntegration.kt} (91%) rename services/s3/common/src/aws/sdk/kotlin/services/s3/internal/{S3ExpressIdentityCache.kt => S3ExpressCredentialsCache.kt} (100%) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index 407e270baf1..026d3787224 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -65,7 +65,7 @@ class ClientConfigIntegration : KotlinIntegration { } val DisableExpressSessionAuth: ConfigProperty = ConfigProperty { - name = "disableExpressSessionAuth" + name = "disableS3ExpressSessionAuth" useSymbolWithNullableBuilder(KotlinTypes.Boolean, "false") documentation = """ Flag to disable S3 Express One Zone's bucket-level session authentication method. diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/ConfigureS3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt similarity index 91% rename from codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/ConfigureS3ExpressAuthSchemeIntegration.kt rename to codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 7e34d3d55c1..9f81af73d11 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/ConfigureS3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -25,7 +25,14 @@ import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ServiceShape import software.amazon.smithy.model.shapes.StructureShape -class ConfigureS3ExpressAuthSchemeIntegration : KotlinIntegration { +/** + * An integration which sets up multiple code-generation aspects for S3 Express. + * 1. Configure auth scheme (auth scheme ID, auth option, identity provider for auth scheme) + * 2. Add S3Client and Bucket to execution context (required for the S3 Express credentials cache key) + * 3. Override all checksums to use CRC32 instead + * 4. Disable all checksums for s3:UploadPart + */ +class S3ExpressIntegration : KotlinIntegration { override fun enabledForService(model: Model, settings: KotlinSettings) = model.expectShape(settings.service).isS3 override val sectionWriters: List diff --git a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration index 9702bd5c869..71be3e4feae 100644 --- a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +++ b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration @@ -37,4 +37,4 @@ aws.sdk.kotlin.codegen.customization.route53.ChangeResourceRecordSetsUnmarshalli aws.sdk.kotlin.codegen.customization.ec2.EC2MakePrimitivesOptional aws.sdk.kotlin.codegen.customization.RemoveDefaults aws.sdk.kotlin.codegen.customization.s3.express.SigV4S3ExpressAuthSchemeIntegration -aws.sdk.kotlin.codegen.customization.s3.express.ConfigureS3ExpressAuthSchemeIntegration +aws.sdk.kotlin.codegen.customization.s3.express.S3ExpressIntegration diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt index 3e6a03fb594..f3e2f2ae43b 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/FinalizeS3Config.kt @@ -20,7 +20,7 @@ internal suspend fun finalizeS3Config( val activeProfile = sharedConfig.get().activeProfile builder.config.useArnRegion = builder.config.useArnRegion ?: S3Setting.UseArnRegion.resolve(provider) ?: activeProfile.useArnRegion builder.config.disableMrap = builder.config.disableMrap ?: S3Setting.DisableMultiRegionAccessPoints.resolve(provider) ?: activeProfile.disableMrap - builder.config.disableExpressSessionAuth = builder.config.disableExpressSessionAuth ?: S3Setting.DisableExpressSessionAuth.resolve(provider) ?: activeProfile.disableExpressSessionAuth + builder.config.disableS3ExpressSessionAuth = builder.config.disableS3ExpressSessionAuth ?: S3Setting.DisableS3ExpressSessionAuth.resolve(provider) ?: activeProfile.disableS3ExpressSessionAuth } private val AwsProfile.useArnRegion: Boolean? @@ -29,5 +29,5 @@ private val AwsProfile.useArnRegion: Boolean? private val AwsProfile.disableMrap: Boolean? get() = getBooleanOrNull("s3_disable_multiregion_access_points") -private val AwsProfile.disableExpressSessionAuth: Boolean? +private val AwsProfile.disableS3ExpressSessionAuth: Boolean? get() = getBooleanOrNull("s3_disable_express_session_auth") diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressIdentityCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt similarity index 100% rename from services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressIdentityCache.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt index 5ad4b4b088a..38df137adb7 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt @@ -5,8 +5,6 @@ package aws.sdk.kotlin.services.s3.internal import aws.sdk.kotlin.runtime.auth.S3ExpressAttributes -import aws.sdk.kotlin.runtime.auth.credentials.internal.S3ExpressCredentialsCache -import aws.sdk.kotlin.runtime.auth.credentials.internal.S3ExpressCredentialsCacheKey import aws.sdk.kotlin.runtime.auth.credentials.internal.credentials import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.model.CreateSessionRequest diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3Setting.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3Setting.kt index 34746e677b0..57c4d63bd79 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3Setting.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3Setting.kt @@ -25,5 +25,5 @@ internal object S3Setting { /** * Configure whether requests made to S3 Express One Zone should use bucket-level session authentication or the default S3 authentication method. */ - public val DisableExpressSessionAuth: EnvironmentSetting = boolEnvSetting("aws.s3DisableExpressSessionAuth", "AWS_S3_DISABLE_EXPRESS_SESSION_AUTH") + public val DisableS3ExpressSessionAuth: EnvironmentSetting = boolEnvSetting("aws.s3DisableExpressSessionAuth", "AWS_S3_DISABLE_EXPRESS_SESSION_AUTH") } From aaa342ec4b3b3858953f09e460ea5ac7033eb2df Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 12 Jan 2024 14:43:00 -0500 Subject: [PATCH 007/145] checksums --- aws-runtime/aws-http/api/aws-http.api | 49 ++++++++++++++ .../S3ExpressCrc32ChecksumInterceptor.kt | 38 +++++++++++ .../S3ExpressDisableChecksumInterceptor.kt | 31 +++++++++ .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 2 + .../FlexibleChecksumsRequest.kt | 11 +--- .../s3/express/S3ExpressCRC32Checksum.kt | 5 -- .../s3/express/S3ExpressIntegration.kt | 65 +++++++++++++++++-- 7 files changed, 184 insertions(+), 17 deletions(-) create mode 100644 aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt create mode 100644 aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt delete mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressCRC32Checksum.kt diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index 3e15d87841f..73c1ae019bd 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -37,6 +37,55 @@ public final class aws/sdk/kotlin/runtime/http/AwsUserAgentMetadataKt { public static final field AWS_APP_ID_PROP Ljava/lang/String; } +public final class aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { + public fun ()V + public fun (Ljava/lang/String;)V + public synthetic fun (Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getChecksumAlgorithmHeaderName ()Ljava/lang/String; + public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeRetryLoop (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun readAfterAttempt (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterDeserialization (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterExecution (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterSerialization (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readAfterSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readAfterTransmit (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V + public fun readBeforeAttempt (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V + public fun readBeforeExecution (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V + public fun readBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V + public fun readBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V +} + +public final class aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { + public fun ()V + public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeRetryLoop (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun modifyBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun readAfterAttempt (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterDeserialization (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterExecution (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V + public fun readAfterSerialization (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readAfterSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readAfterTransmit (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V + public fun readBeforeAttempt (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V + public fun readBeforeExecution (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V + public fun readBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V + public fun readBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V + public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V +} + public final class aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadata { public fun ()V public fun (Ljava/util/Map;Ljava/util/List;)V diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt new file mode 100644 index 00000000000..b22e7434ad0 --- /dev/null +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt @@ -0,0 +1,38 @@ +package aws.sdk.kotlin.runtime.http.interceptors + +import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.collections.AttributeKey +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.request.toBuilder +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlin.coroutines.coroutineContext + +internal val S3_EXPRESS_ENDPOINT_PROPERTY = "backend" +private val CRC32_ALGORITHM_NAME = "CRC32" + +public class S3ExpressCrc32ChecksumInterceptor( + public val checksumAlgorithmHeaderName: String? = null +): HttpInterceptor { + override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { + if (!context.executionContext.contains(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY))) { + return context.protocolRequest + } + + val logger = coroutineContext.logger() + val req = context.protocolRequest.toBuilder() + + // Update the execution context so flexible checksums uses CRC32 + logger.info { "Setting checksum algorithm to $CRC32_ALGORITHM_NAME for S3 Express" } + context.executionContext[HttpOperationContext.ChecksumAlgorithm] = CRC32_ALGORITHM_NAME + + // Most checksum headers are handled by the flexible checksums feature. But,services may model an HTTP header binding for the + // checksum algorithm, which also needs to be overwritten and set to CRC32. + checksumAlgorithmHeaderName?.let { + req.headers[it] = CRC32_ALGORITHM_NAME + } + + return req.build() + } +} \ No newline at end of file diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt new file mode 100644 index 00000000000..9ee36e67262 --- /dev/null +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt @@ -0,0 +1,31 @@ +package aws.sdk.kotlin.runtime.http.interceptors + +import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.collections.AttributeKey +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlin.coroutines.coroutineContext + +/** + * Disable checksums entirely for s3:UploadPart requests. + */ +public class S3ExpressDisableChecksumInterceptor : HttpInterceptor { + override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { + if (!context.executionContext.contains(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY))) { + return context.protocolRequest + } + + val logger = coroutineContext.logger() + + val configuredChecksumAlgorithm = context.executionContext.getOrNull(HttpOperationContext.ChecksumAlgorithm) + + configuredChecksumAlgorithm?.let { + logger.info { "Disabling configured checksum $it for S3 Express" } + context.executionContext.remove(HttpOperationContext.ChecksumAlgorithm) + } + + return context.protocolRequest + } +} \ No newline at end of file diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index bb6584f3098..c522b45c12c 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -65,6 +65,8 @@ object AwsRuntimeTypes { object Http : RuntimeTypePackage(AwsKotlinDependency.AWS_HTTP) { object Interceptors : RuntimeTypePackage(AwsKotlinDependency.AWS_HTTP, "interceptors") { val AddUserAgentMetadataInterceptor = symbol("AddUserAgentMetadataInterceptor") + val S3ExpressCrc32ChecksumInterceptor = symbol("S3ExpressCrc32ChecksumInterceptor") + val S3ExpressDisableChecksumInterceptor = symbol("S3ExpressDisableChecksumInterceptor") } object Retries { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index d13aa4eb3c2..74b0ae0e654 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -43,7 +43,6 @@ class FlexibleChecksumsRequest : KotlinIntegration { } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) val interceptorSymbol = RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor val httpChecksumTrait = op.getTrait()!! @@ -52,13 +51,9 @@ class FlexibleChecksumsRequest : KotlinIntegration { .members() .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } - writer.withBlock( - "op.interceptors.add(#T<#T> {", - "})", - interceptorSymbol, - inputSymbol, - ) { - writer.write("it.#L?.value", requestAlgorithmMember.defaultName()) + writer.write("op.interceptors.add(#T())", interceptorSymbol) + writer.withBlock("input.${requestAlgorithmMember.defaultName()}?.let {", "}") { + writer.write("op.context[#T.ChecksumAlgorithm] = it.value", RuntimeTypes.HttpClient.Operation.HttpOperationContext) } } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressCRC32Checksum.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressCRC32Checksum.kt deleted file mode 100644 index d858e0d42ab..00000000000 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressCRC32Checksum.kt +++ /dev/null @@ -1,5 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 9f81af73d11..bdb046f129c 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -7,23 +7,28 @@ package aws.sdk.kotlin.codegen.customization.s3.express import aws.sdk.kotlin.codegen.AwsRuntimeTypes import aws.sdk.kotlin.codegen.customization.s3.ClientConfigIntegration import aws.sdk.kotlin.codegen.customization.s3.isS3 +import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings -import software.amazon.smithy.kotlin.codegen.core.KotlinWriter -import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes -import software.amazon.smithy.kotlin.codegen.core.withBlock +import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding import software.amazon.smithy.kotlin.codegen.model.expectShape +import software.amazon.smithy.kotlin.codegen.model.getTrait +import software.amazon.smithy.kotlin.codegen.model.hasTrait import software.amazon.smithy.kotlin.codegen.rendering.auth.AuthSchemeProviderGenerator import software.amazon.smithy.kotlin.codegen.rendering.auth.IdentityProviderConfigGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpProtocolClientGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.kotlin.codegen.utils.dq +import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ServiceShape import software.amazon.smithy.model.shapes.StructureShape +import software.amazon.smithy.model.traits.HttpChecksumRequiredTrait +import software.amazon.smithy.model.traits.HttpHeaderTrait /** * An integration which sets up multiple code-generation aspects for S3 Express. @@ -57,7 +62,7 @@ class S3ExpressIntegration : KotlinIntegration { } override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + AddClientToExecutionContext + AddBucketToExecutionContext + resolved + AddClientToExecutionContext + AddBucketToExecutionContext + UseCrc32Checksum + UploadPartDisableChecksum private val AddClientToExecutionContext = object : ProtocolMiddleware { override val name: String = "AddClientToExecutionContext" @@ -83,4 +88,56 @@ class S3ExpressIntegration : KotlinIntegration { writer.write("input.bucket?.let { op.context[#T.Bucket] = it }", attributesSymbol) } } + + /** + * For any operations that may send a checksum, override a user-configured checksum to set CRC32. + */ + private val UseCrc32Checksum = object : ProtocolMiddleware { + override val name: String = "UseCrc32Checksum" + + override val order: Byte = -1 // Render before flexible checksums + + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + !op.isS3UploadPart && (op.hasTrait() || op.hasTrait()) + + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val httpChecksumTrait = op.getTrait() + + val checksumAlgorithmMember = ctx.model.expectShape(op.input.get()) + .members() + .firstOrNull { it.memberName == httpChecksumTrait?.requestAlgorithmMember?.getOrNull() } + + val checksumHeaderName = checksumAlgorithmMember?.getTrait()?.value + + val checksumRequiredTrait = op.getTrait() + + if (checksumAlgorithmMember != null) { + if (checksumRequiredTrait != null) { // checksum required, enable flexible checksums using CRC32 + writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq()}))", AwsRuntimeTypes.Http.Interceptors.S3ExpressCrc32ChecksumInterceptor) + } else { // checksum not required, override with CRC32 only if user enabled flexible checksums + writer.withBlock("if (input.${checksumAlgorithmMember.defaultName()} != null) {", "}") { + writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq()}))", AwsRuntimeTypes.Http.Interceptors.S3ExpressCrc32ChecksumInterceptor) + } + } + } else { + if (checksumRequiredTrait != null) { + error("Checksum is required but operation does not support flexible checksums. Can't proceed because S3 Express requires sending CRC32 checksums, not MD5.") + } + } + } + } + + private val UploadPartDisableChecksum = object : ProtocolMiddleware { + override val name: String = "UploadPartDisableChecksum" + + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = op.isS3UploadPart + + override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val interceptorSymbol = AwsRuntimeTypes.Http.Interceptors.S3ExpressDisableChecksumInterceptor + writer.addImport(interceptorSymbol) + writer.write("op.interceptors.add(#T())", interceptorSymbol) + } + } + + private val OperationShape.isS3UploadPart: Boolean get() = id.name == "UploadPart" } From 222ea548ce4339ff93322b750cd56990918eba5d Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 12 Jan 2024 14:43:37 -0500 Subject: [PATCH 008/145] ktlintFormat --- .../interceptors/S3ExpressCrc32ChecksumInterceptor.kt | 10 +++++++--- .../S3ExpressDisableChecksumInterceptor.kt | 6 +++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt index b22e7434ad0..527c2f3e179 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt @@ -1,3 +1,7 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ package aws.sdk.kotlin.runtime.http.interceptors import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext @@ -13,8 +17,8 @@ internal val S3_EXPRESS_ENDPOINT_PROPERTY = "backend" private val CRC32_ALGORITHM_NAME = "CRC32" public class S3ExpressCrc32ChecksumInterceptor( - public val checksumAlgorithmHeaderName: String? = null -): HttpInterceptor { + public val checksumAlgorithmHeaderName: String? = null, +) : HttpInterceptor { override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { if (!context.executionContext.contains(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY))) { return context.protocolRequest @@ -35,4 +39,4 @@ public class S3ExpressCrc32ChecksumInterceptor( return req.build() } -} \ No newline at end of file +} diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt index 9ee36e67262..e5d6b7f4f25 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt @@ -1,3 +1,7 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ package aws.sdk.kotlin.runtime.http.interceptors import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext @@ -28,4 +32,4 @@ public class S3ExpressDisableChecksumInterceptor : HttpInterceptor { return context.protocolRequest } -} \ No newline at end of file +} From d030ffa67248fd1d647e164884da3be6e1210fc9 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 16 Jan 2024 10:34:13 -0500 Subject: [PATCH 009/145] Upgrade smithy-kotlin --- gradle/libs.versions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0ff91325cbd..fc2e2d1b38d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -9,8 +9,8 @@ coroutines-version = "1.7.3" atomicfu-version = "0.23.1" # smithy-kotlin codegen and runtime are versioned separately -smithy-kotlin-runtime-version = "1.0.8" -smithy-kotlin-codegen-version = "0.30.9" +smithy-kotlin-runtime-version = "1.0.10-SNAPSHOT" +smithy-kotlin-codegen-version = "0.30.11-SNAPSHOT" # codegen smithy-version = "1.42.0" From 441cf2667a0a1444c8c88ec76b378d324ba485a0 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 25 Jan 2024 12:09:04 -0500 Subject: [PATCH 010/145] Asynchronous credentials refresh --- .../SigV4S3ExpressAuthSchemeIntegration.kt | 10 ++ .../s3/internal/S3ExpressCredentialsCache.kt | 109 +++++++++++++++++- .../internal/S3ExpressCredentialsProvider.kt | 30 ++--- 3 files changed, 121 insertions(+), 28 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index 2fddcca6dd0..b517a81b46c 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -12,7 +12,9 @@ import software.amazon.smithy.codegen.core.Symbol import software.amazon.smithy.codegen.core.SymbolReference import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.* +import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter import software.amazon.smithy.kotlin.codegen.integration.AuthSchemeHandler +import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding import software.amazon.smithy.kotlin.codegen.model.buildSymbol import software.amazon.smithy.kotlin.codegen.model.expectShape import software.amazon.smithy.kotlin.codegen.model.hasTrait @@ -40,6 +42,14 @@ class SigV4S3ExpressAuthSchemeIntegration : SigV4AuthSchemeIntegration() { override fun customizeEndpointResolution(ctx: ProtocolGenerator.GenerationContext): EndpointCustomization = SigV4S3ExpressEndpointCustomization + override val sectionWriters: List + get() = listOf(SectionWriterBinding(HttpProtocolClientGenerator.ClientInitializer, renderClientInitializer)) + + // add S3 Express credentials provider to managed resources in the service client initializer + private val renderClientInitializer = AppendingSectionWriter { writer -> + writer.write("managedResources.addIfManaged(config.s3ExpressCredentialsProvider)") + } + override fun authSchemes(ctx: ProtocolGenerator.GenerationContext): List = listOf(SigV4S3ExpressAuthSchemeHandler()) } diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 1df09e01093..6b0693d327d 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -9,20 +9,117 @@ import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.collections.LruCache import aws.smithy.kotlin.runtime.time.Clock import aws.smithy.kotlin.runtime.util.ExpiringValue +import aws.smithy.kotlin.runtime.io.Closeable +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.* +import kotlinx.coroutines.selects.* +import kotlin.coroutines.coroutineContext +import aws.sdk.kotlin.services.s3.* +import aws.sdk.kotlin.services.s3.model.CreateSessionRequest +import aws.sdk.kotlin.runtime.auth.credentials.internal.credentials +import kotlin.time.Duration.Companion.minutes +import kotlin.coroutines.CoroutineContext +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlin.time.Duration +import kotlin.time.* +import aws.smithy.kotlin.runtime.time.until private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 +private val REFRESH_BUFFER = 1.minutes +private val DEFAULT_REFRESH_PERIOD = 5.minutes public class S3ExpressCredentialsCache( private val clock: Clock = Clock.System, -) { +) : CoroutineScope, Closeable { + override val coroutineContext: CoroutineContext = Job() + CoroutineName("S3ExpressCredentialsCacheRefresh") + private val lru = LruCache>(DEFAULT_S3_EXPRESS_CACHE_SIZE) + private val immediateRefreshChannel = Channel(Channel.CONFLATED) // channel used to indicate an immediate refresh attempt is required + + init { + launch(coroutineContext) { + refresh() + } + } + + public suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru.get(key)?.value + ?: (createSessionCredentials(key).also { put(key, it) }).value + + public suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue): Unit { + lru.put(key, value) + immediateRefreshChannel.send(Unit) + } + + private suspend fun refresh(): Unit { + val logger = coroutineContext.logger() + while (isActive) { + logger.trace { "Looping..." } + println("Looping...") + val refreshedCredentials = mutableMapOf>() + var nextRefresh: Instant = clock.now() + DEFAULT_REFRESH_PERIOD + + lru.withLock { + lru.entries.forEach { (key, cachedValue) -> + logger.trace { "Checking entry for ${key.bucket}" } + println("Checking entry for ${key.bucket}") + nextRefresh = minOf(nextRefresh, cachedValue.expiresAt) - public suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials? = ( - lru.get(key) - ?.takeIf { it.expiresAt > clock.now() } - )?.value + if ((clock.now().until(cachedValue.expiresAt)).absoluteValue <= REFRESH_BUFFER) { + logger.trace { "Credentials for ${key.bucket} expire within the refresh buffer period, performing a refresh..." } + println("Credentials for ${key.bucket} expire within the refresh buffer period, performing a refresh...") + createSessionCredentials(key).also { + refreshedCredentials.put(key, it) + nextRefresh = minOf(nextRefresh, it.expiresAt) + } + } + } - public suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue): Unit = lru.put(key, value) + refreshedCredentials.forEach { (key, value) -> + lru.remove(key) + lru.putUnlocked(key, value) + } + } + + // wake up when it's time to refresh or an immediate refresh has been triggered + select { + onTimeout(clock.now().until(nextRefresh)) { + logger.trace { "Woke up from timeout" } + println("Woke up from timeout") + } + immediateRefreshChannel.onReceive { + logger.trace { "Woke up from channel" } + println("Woke up from channel") + } + } + } + } + + private suspend fun createSessionCredentials(key: S3ExpressCredentialsCacheKey): ExpiringValue { + val logger = coroutineContext.logger() + + val credentials = (key.client as S3Client).createSession( + CreateSessionRequest { + bucket = key.bucket + }, + ).credentials!! + + return ExpiringValue( + credentials( + accessKeyId = credentials.accessKeyId, + secretAccessKey = credentials.secretAccessKey, + sessionToken = credentials.sessionToken, + expiration = credentials.expiration, + providerName = "S3ExpressCredentialsProvider", + ), + credentials.expiration, + ).also { logger.debug { "got credentials ${it.value}" } } + } + + override fun close(): Unit { + coroutineContext.cancel(null) + immediateRefreshChannel.close() + } } public class S3ExpressCredentialsCacheKey( diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt index 38df137adb7..1aa7f3442b8 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt @@ -10,45 +10,31 @@ import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.model.CreateSessionRequest import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider +import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvider import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.telemetry.logging.logger import aws.smithy.kotlin.runtime.util.ExpiringValue import kotlin.coroutines.coroutineContext +import aws.smithy.kotlin.runtime.io.SdkManagedBase public class S3ExpressCredentialsProvider( public val bootstrapCredentialsProvider: CredentialsProvider, -) : CredentialsProvider { +) : CloseableCredentialsProvider, SdkManagedBase() { private val credentialsCache = S3ExpressCredentialsCache() override suspend fun resolve(attributes: Attributes): Credentials { + val logger = coroutineContext.logger() + val bucket: String = attributes[S3ExpressAttributes.Bucket] val client = attributes[S3ExpressAttributes.Client] val key = S3ExpressCredentialsCacheKey(bucket, client, bootstrapCredentialsProvider.resolve(attributes)) - return credentialsCache.get(key) - ?: (createSessionCredentials(key).also { credentialsCache.put(key, it) }).value + return credentialsCache.get(key).also { logger.trace { "Got credentials $it from cache" }} } - private suspend fun createSessionCredentials(key: S3ExpressCredentialsCacheKey): ExpiringValue { - val logger = coroutineContext.logger() - - val credentials = (key.client as S3Client).createSession( - CreateSessionRequest { - bucket = key.bucket - }, - ).credentials!! - - return ExpiringValue( - credentials( - accessKeyId = credentials.accessKeyId, - secretAccessKey = credentials.secretAccessKey, - sessionToken = credentials.sessionToken, - expiration = credentials.expiration, - providerName = "S3ExpressCredentialsProvider", - ), - credentials.expiration, - ).also { logger.debug { "got credentials ${it.value}" } } + override fun close() { + credentialsCache.close() } } From d24741425b187cb9641b33749cce39609d6c7b46 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Mon, 5 Feb 2024 17:09:52 -0500 Subject: [PATCH 011/145] Bump to latest SNAPSHOT version --- gradle/libs.versions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1d2afe76a04..227fe8c8da6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -9,8 +9,8 @@ coroutines-version = "1.7.3" atomicfu-version = "0.23.1" # smithy-kotlin codegen and runtime are versioned separately -smithy-kotlin-runtime-version = "1.0.12-SNAPSHOT" -smithy-kotlin-codegen-version = "0.30.13-SNAPSHOT" +smithy-kotlin-runtime-version = "1.0.13-SNAPSHOT" +smithy-kotlin-codegen-version = "0.30.14-SNAPSHOT" # codegen smithy-version = "1.42.0" From e679bcdbd1df8f10ffee9b7369f04d3a092e7db7 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Mon, 5 Feb 2024 17:32:03 -0500 Subject: [PATCH 012/145] lint --- .../customization/s3/express/S3ExpressIntegration.kt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index bdb046f129c..1fb92acb5c6 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -62,7 +62,12 @@ class S3ExpressIntegration : KotlinIntegration { } override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = - resolved + AddClientToExecutionContext + AddBucketToExecutionContext + UseCrc32Checksum + UploadPartDisableChecksum + resolved + listOf( + AddClientToExecutionContext, + AddBucketToExecutionContext, + UseCrc32Checksum, + UploadPartDisableChecksum, + ) private val AddClientToExecutionContext = object : ProtocolMiddleware { override val name: String = "AddClientToExecutionContext" From 0c85adf0bd3c3a14f7388c480ca1302d2a83e698 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Mon, 5 Feb 2024 17:32:16 -0500 Subject: [PATCH 013/145] api --- aws-runtime/aws-config/api/aws-config.api | 2 +- aws-runtime/aws-http/api/aws-http.api | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index b9d5aa3a95b..21ef9846ec4 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -7,7 +7,7 @@ public final class aws/sdk/kotlin/runtime/auth/S3ExpressAttributes { public final class aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme : aws/smithy/kotlin/runtime/http/auth/AuthScheme { public fun (Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner;Ljava/lang/String;)V public synthetic fun (Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V - public fun (Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner$Config;)V + public fun (Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner;)V public fun getSchemeId-DepwgT4 ()Ljava/lang/String; public fun getSigner ()Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner; public synthetic fun getSigner ()Laws/smithy/kotlin/runtime/http/auth/HttpSigner; diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index 73c1ae019bd..aca809d92e5 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -37,6 +37,12 @@ public final class aws/sdk/kotlin/runtime/http/AwsUserAgentMetadataKt { public static final field AWS_APP_ID_PROP Ljava/lang/String; } +public final class aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner : aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner { + public fun (Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner;)V + public final fun getAwsHttpSigner ()Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner; + public fun sign (Laws/smithy/kotlin/runtime/http/auth/SignHttpRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; +} + public final class aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { public fun ()V public fun (Ljava/lang/String;)V From 1067774d175018c7e1ddad4b7bafdb6e7a283ada Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 10:01:51 -0500 Subject: [PATCH 014/145] fix checksums --- .../S3ExpressCrc32ChecksumInterceptor.kt | 5 ++++- .../s3/express/S3ExpressIntegration.kt | 13 ++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt index 527c2f3e179..5b323ff0596 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt @@ -31,8 +31,11 @@ public class S3ExpressCrc32ChecksumInterceptor( logger.info { "Setting checksum algorithm to $CRC32_ALGORITHM_NAME for S3 Express" } context.executionContext[HttpOperationContext.ChecksumAlgorithm] = CRC32_ALGORITHM_NAME - // Most checksum headers are handled by the flexible checksums feature. But,services may model an HTTP header binding for the + // Most checksum headers are handled by the flexible checksums feature. But, S3 models an HTTP header binding for the // checksum algorithm, which also needs to be overwritten and set to CRC32. + // + // The header is already set by the time this interceptor runs, so it needs to be overwritten and can't be set + // through the normal path. checksumAlgorithmHeaderName?.let { req.headers[it] = CRC32_ALGORITHM_NAME } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 1fb92acb5c6..c1e04b08665 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -100,7 +100,7 @@ class S3ExpressIntegration : KotlinIntegration { private val UseCrc32Checksum = object : ProtocolMiddleware { override val name: String = "UseCrc32Checksum" - override val order: Byte = -1 // Render before flexible checksums + override val order: Byte = 1 // Render after flexible checksums override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = !op.isS3UploadPart && (op.hasTrait() || op.hasTrait()) @@ -112,21 +112,20 @@ class S3ExpressIntegration : KotlinIntegration { .members() .firstOrNull { it.memberName == httpChecksumTrait?.requestAlgorithmMember?.getOrNull() } + // S3 models a header name x-amz-sdk-checksum-algorithm representing the name of the checksum algorithm used val checksumHeaderName = checksumAlgorithmMember?.getTrait()?.value - val checksumRequiredTrait = op.getTrait() + val checksumRequired = op.hasTrait() || httpChecksumTrait?.isRequestChecksumRequired == true if (checksumAlgorithmMember != null) { - if (checksumRequiredTrait != null) { // checksum required, enable flexible checksums using CRC32 - writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq()}))", AwsRuntimeTypes.Http.Interceptors.S3ExpressCrc32ChecksumInterceptor) - } else { // checksum not required, override with CRC32 only if user enabled flexible checksums + if (checksumRequired) { // checksum required, enable flexible checksums using CRC32 if the user has not already opted-in writer.withBlock("if (input.${checksumAlgorithmMember.defaultName()} != null) {", "}") { writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq()}))", AwsRuntimeTypes.Http.Interceptors.S3ExpressCrc32ChecksumInterceptor) } } } else { - if (checksumRequiredTrait != null) { - error("Checksum is required but operation does not support flexible checksums. Can't proceed because S3 Express requires sending CRC32 checksums, not MD5.") + if (checksumRequired) { // checksum required, operation does not support flexible checksums, so set the checksum algorithm manually + writer.write("op.interceptors.add(#T())", AwsRuntimeTypes.Http.Interceptors.S3ExpressCrc32ChecksumInterceptor) } } } From f6b67d2c1b0aa4eae47f916925591c01b84e3af2 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 10:08:11 -0500 Subject: [PATCH 015/145] fix checksums --- .../codegen/customization/s3/express/S3ExpressIntegration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index c1e04b08665..bad517fabc6 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -100,7 +100,7 @@ class S3ExpressIntegration : KotlinIntegration { private val UseCrc32Checksum = object : ProtocolMiddleware { override val name: String = "UseCrc32Checksum" - override val order: Byte = 1 // Render after flexible checksums + override val order: Byte = -1 // Render before flexible checksums override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = !op.isS3UploadPart && (op.hasTrait() || op.hasTrait()) From 78fb58c6e41d35787f2812c73cb44ebc946407d4 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 12:13:43 -0500 Subject: [PATCH 016/145] Fixed session token header --- aws-runtime/aws-config/api/aws-config.api | 3 +- .../runtime/auth/SigV4S3ExpressAuthScheme.kt | 12 ++++--- aws-runtime/aws-http/api/aws-http.api | 2 +- .../runtime/http/S3ExpressHttpSigner.kt | 35 +++++++++++++++++++ 4 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index 21ef9846ec4..e662d3ab747 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -9,8 +9,7 @@ public final class aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme : aws/sm public synthetic fun (Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public fun (Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner;)V public fun getSchemeId-DepwgT4 ()Ljava/lang/String; - public fun getSigner ()Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner; - public synthetic fun getSigner ()Laws/smithy/kotlin/runtime/http/auth/HttpSigner; + public fun getSigner ()Laws/smithy/kotlin/runtime/http/auth/HttpSigner; public fun identityProvider (Laws/smithy/kotlin/runtime/identity/IdentityProviderConfig;)Laws/smithy/kotlin/runtime/identity/IdentityProvider; } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt index 866272ce92d..c5b46bae52f 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt @@ -5,6 +5,7 @@ package aws.sdk.kotlin.runtime.auth +import aws.sdk.kotlin.runtime.http.S3ExpressHttpSigner import aws.smithy.kotlin.runtime.InternalApi import aws.smithy.kotlin.runtime.auth.AuthOption import aws.smithy.kotlin.runtime.auth.AuthSchemeId @@ -18,23 +19,24 @@ import aws.smithy.kotlin.runtime.collections.emptyAttributes import aws.smithy.kotlin.runtime.collections.mutableAttributes import aws.smithy.kotlin.runtime.http.auth.AuthScheme import aws.smithy.kotlin.runtime.http.auth.AwsHttpSigner +import aws.smithy.kotlin.runtime.http.auth.HttpSigner /** * HTTP auth scheme for S3 Express One Zone authentication */ public class SigV4S3ExpressAuthScheme( - config: AwsHttpSigner.Config, + awsHttpSigner: AwsHttpSigner, ) : AuthScheme { - public constructor(awsSigner: AwsSigner, serviceName: String? = null) : this( + public constructor(awsSigner: AwsSigner, serviceName: String? = null) : this(AwsHttpSigner( AwsHttpSigner.Config().apply { signer = awsSigner service = serviceName algorithm = AwsSigningAlgorithm.SIGV4_S3EXPRESS - }, - ) + } + )) override val schemeId: AuthSchemeId = AuthSchemeId.AwsSigV4S3Express - override val signer: AwsHttpSigner = AwsHttpSigner(config) + override val signer: HttpSigner = S3ExpressHttpSigner(awsHttpSigner) } /** diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index aca809d92e5..457091821c1 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -37,7 +37,7 @@ public final class aws/sdk/kotlin/runtime/http/AwsUserAgentMetadataKt { public static final field AWS_APP_ID_PROP Ljava/lang/String; } -public final class aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner : aws/smithy/kotlin/runtime/http/auth/AwsHttpSigner { +public final class aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner : aws/smithy/kotlin/runtime/http/auth/HttpSigner { public fun (Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner;)V public final fun getAwsHttpSigner ()Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner; public fun sign (Laws/smithy/kotlin/runtime/http/auth/SignHttpRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt new file mode 100644 index 00000000000..7d1cf83aea8 --- /dev/null +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt @@ -0,0 +1,35 @@ +package aws.sdk.kotlin.runtime.http + +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAttributes +import aws.smithy.kotlin.runtime.collections.toMutableAttributes +import aws.smithy.kotlin.runtime.http.auth.AwsHttpSigner +import aws.smithy.kotlin.runtime.http.auth.HttpSigner +import aws.smithy.kotlin.runtime.http.auth.SignHttpRequest +import aws.smithy.kotlin.runtime.http.request.header + +private const val S3_EXPRESS_SESSION_TOKEN_HEADER = "X-Amz-S3Session-Token" +private const val SESSION_TOKEN_HEADER = "X-Amz-Security-Token" + +public class S3ExpressHttpSigner( + public val awsHttpSigner: AwsHttpSigner +): HttpSigner { + public override suspend fun sign(signingRequest: SignHttpRequest) { + val sessionToken = (signingRequest.identity as? Credentials)?.sessionToken + ?: error("S3ExpressHttpSigner failed to parse sessionToken from identity") + + // 1. add the S3 Express Session Token header + signingRequest.httpRequest.header(S3_EXPRESS_SESSION_TOKEN_HEADER, sessionToken) + println("Adding session token $sessionToken") + + // 2. enable omitSessionToken for awsHttpSigner + val mutAttrs = signingRequest.signingAttributes.toMutableAttributes() + mutAttrs[AwsSigningAttributes.OmitSessionToken] = true + + // 3. call main signer + awsHttpSigner.sign(signingRequest.copy(signingAttributes = mutAttrs)) + + // 4. remove standard session token header + signingRequest.httpRequest.headers.remove(SESSION_TOKEN_HEADER) + } +} \ No newline at end of file From 4e4cdf01f17a66362fe51cb0fc0a07b573a87f55 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 12:45:34 -0500 Subject: [PATCH 017/145] KDocs --- .../kotlin/runtime/http/S3ExpressHttpSigner.kt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt index 7d1cf83aea8..ac6af0d4811 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt @@ -11,25 +11,33 @@ import aws.smithy.kotlin.runtime.http.request.header private const val S3_EXPRESS_SESSION_TOKEN_HEADER = "X-Amz-S3Session-Token" private const val SESSION_TOKEN_HEADER = "X-Amz-Security-Token" +/** + * An [HttpSigner] used for S3 Express requests. It wraps [AwsHttpSigner] and has identical behavior except for two differences: + * 1. Adds an `X-Amz-S3Session-Token` header, with a value of the credentials' sessionToken + * 2. Removes the `X-Amz-Security-Token` header, which must not be sent for S3 Express requests. + * @param awsHttpSigner An instance of [AwsHttpSigner] + */ public class S3ExpressHttpSigner( public val awsHttpSigner: AwsHttpSigner ): HttpSigner { + /** + * Sign the request, adding `X-Amz-S3Session-Token` header and removing `X-Amz-Security-Token` header. + */ public override suspend fun sign(signingRequest: SignHttpRequest) { val sessionToken = (signingRequest.identity as? Credentials)?.sessionToken - ?: error("S3ExpressHttpSigner failed to parse sessionToken from identity") + ?: error("Failed to parse sessionToken from identity") // 1. add the S3 Express Session Token header signingRequest.httpRequest.header(S3_EXPRESS_SESSION_TOKEN_HEADER, sessionToken) - println("Adding session token $sessionToken") - // 2. enable omitSessionToken for awsHttpSigner + // 2. enable omitSessionToken for awsHttpSigner to disable signing session token header val mutAttrs = signingRequest.signingAttributes.toMutableAttributes() mutAttrs[AwsSigningAttributes.OmitSessionToken] = true // 3. call main signer awsHttpSigner.sign(signingRequest.copy(signingAttributes = mutAttrs)) - // 4. remove standard session token header + // 4. remove session token header signingRequest.httpRequest.headers.remove(SESSION_TOKEN_HEADER) } } \ No newline at end of file From a20499726bfc86208345c2e3cdeda7783701e005 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 12:46:41 -0500 Subject: [PATCH 018/145] E2E tests --- .../s3/express/S3ExpressIntegration.kt | 10 +- services/s3/e2eTest/src/S3ExpressTest.kt | 121 ++++++++++++++++++ services/s3/e2eTest/src/S3TestUtils.kt | 45 +++++++ 3 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 services/s3/e2eTest/src/S3ExpressTest.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index bad517fabc6..81a0fda6590 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -117,14 +117,12 @@ class S3ExpressIntegration : KotlinIntegration { val checksumRequired = op.hasTrait() || httpChecksumTrait?.isRequestChecksumRequired == true - if (checksumAlgorithmMember != null) { - if (checksumRequired) { // checksum required, enable flexible checksums using CRC32 if the user has not already opted-in - writer.withBlock("if (input.${checksumAlgorithmMember.defaultName()} != null) {", "}") { + if (checksumRequired) { + if (checksumAlgorithmMember != null) { // enable flexible checksums using CRC32 if the user has not already opted-in + writer.withBlock("if (input.${checksumAlgorithmMember.defaultName()} == null) {", "}") { writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq()}))", AwsRuntimeTypes.Http.Interceptors.S3ExpressCrc32ChecksumInterceptor) } - } - } else { - if (checksumRequired) { // checksum required, operation does not support flexible checksums, so set the checksum algorithm manually + } else { // checksum required, operation does not expose flexible checksums member, so set the checksum algorithm manually writer.write("op.interceptors.add(#T())", AwsRuntimeTypes.Http.Interceptors.S3ExpressCrc32ChecksumInterceptor) } } diff --git a/services/s3/e2eTest/src/S3ExpressTest.kt b/services/s3/e2eTest/src/S3ExpressTest.kt new file mode 100644 index 00000000000..12f50f19cb9 --- /dev/null +++ b/services/s3/e2eTest/src/S3ExpressTest.kt @@ -0,0 +1,121 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.e2etest + +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.internal.S3ExpressCredentialsProvider +import aws.sdk.kotlin.services.s3.internal.SdkS3ExpressCredentialsProvider +import aws.sdk.kotlin.services.s3.model.* +import aws.sdk.kotlin.services.s3.putObject +import aws.sdk.kotlin.services.s3.withConfig +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider +import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.collections.Attributes +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.decodeToString +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.TestInstance +import kotlin.test.* + +/** + * Tests for S3 Express operations + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) + +class S3ExpressTest { + private val client = S3Client { + region = S3TestUtils.DEFAULT_REGION + } + + private val testBuckets: MutableList = mutableListOf() + + @BeforeAll + fun setup(): Unit = runBlocking { + val suffix = "--usw2-az1--x-s3" // us-west-2 availability zone 1 + + // create a few test buckets to test the credentials cache + testBuckets.add(S3TestUtils.getTestDirectoryBucket(client, suffix)) + testBuckets.add(S3TestUtils.getTestDirectoryBucket(client, suffix)) + testBuckets.add(S3TestUtils.getTestDirectoryBucket(client, suffix)) + } + + @AfterAll + fun cleanup(): Unit = runBlocking { + testBuckets.forEach { bucket -> + S3TestUtils.deleteBucketAndAllContents(client, bucket) + } + client.close() + } + + @Test + fun testPutObject() = runTest { + val content = "30 minutes, or it's free!" + val keyName = "express.txt" + + testBuckets.forEach { bucketName -> + client.putObject { + bucket = bucketName + key = keyName + body = ByteStream.fromString(content) + } + + val req = GetObjectRequest { + bucket = bucketName + key = keyName + } + + val respContent = client.getObject(req) { + it.body?.decodeToString() + } + + assertEquals(content, respContent) + } + } + + @Test + fun testChecksums() = runTest { + val bucketName = testBuckets.first() // only need one bucket for this test + + val keysToDelete = listOf("checksums.txt", "delete-me.txt", "dont-forget-about-me.txt") + keysToDelete.forEach { + client.putObject { + bucket = bucketName + key = it + body = ByteStream.fromString("Check out these sums!") + } + } + + client.withConfig { + interceptors += CRC32ChecksumValidatingInterceptor() + }.use { validatingClient -> + // s3:DeleteObjects requires a checksum, even if the user doesn't specify one. + // normally the SDK would default to MD5, but S3 Express must default to CRC32 instead. + val req = DeleteObjectsRequest { + bucket = bucketName + delete = Delete { + objects = keysToDelete.map { + ObjectIdentifier { key = it } + } + } + } + + validatingClient.deleteObjects(req) + } + } + + private class CRC32ChecksumValidatingInterceptor: HttpInterceptor { + override fun readAfterSigning(context: ProtocolRequestInterceptorContext) { + val headers = context.protocolRequest.headers + assertTrue(headers.contains("x-amz-checksum-crc32"), "Failed to find x-amz-checksum-crc32 header") + assertFalse(headers.contains("Content-MD5"), "Unexpectedly found Content-MD5 header") + } + } +} \ No newline at end of file diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 7ceb34708a4..2dbf720492c 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -9,6 +9,7 @@ import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.paginators.listObjectsV2Paginated import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketExists import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.text.ensurePrefix import kotlinx.coroutines.* import kotlinx.coroutines.flow.* import java.io.OutputStreamWriter @@ -23,8 +24,52 @@ object S3TestUtils { private const val TEST_BUCKET_PREFIX = "s3-test-bucket-" + private const val S3_MAX_BUCKET_NAME_LENGTH = 63 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html + private const val S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX = "--x-s3" + suspend fun getTestBucket(client: S3Client): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX) + suspend fun getTestDirectoryBucket(client: S3Client, suffix: String) = getDirectoryBucket(client, suffix) + + private suspend fun getDirectoryBucket(client: S3Client, suffix: String): String = withTimeout(60.seconds) { + var testBucket = client.listBuckets() + .buckets + ?.mapNotNull { it.name } + ?.firstOrNull { it.startsWith(TEST_BUCKET_PREFIX) && it.endsWith(S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX) } + + if (testBucket == null) { + // Adding S3 Express suffix surpasses the bucket name length limit... trim the UUID if needed + testBucket = TEST_BUCKET_PREFIX + + UUID.randomUUID().toString().subSequence(0 until (S3_MAX_BUCKET_NAME_LENGTH - TEST_BUCKET_PREFIX.length - suffix.ensurePrefix("--").length)) + + suffix.ensurePrefix("--") + + println("Creating S3 bucket: $testBucket") + + val availabilityZone = testBucket // s3-test-bucket-UUID--use1-az4--x-s3 + .removeSuffix(S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX) // s3-test-bucket-UUID--use1-az4 + .substringAfterLast("--") // use1-az4 + + client.createBucket { + bucket = testBucket + createBucketConfiguration { + location = LocationInfo { + type = LocationType.AvailabilityZone + name = availabilityZone + } + bucket = BucketInfo { + type = BucketType.Directory + dataRedundancy = DataRedundancy.SingleAvailabilityZone + } + } + } + } + + // FIXME skipping putBucketLifecycleConfiguration because the following exception is thrown from S3 if I try to do it: + // aws.sdk.kotlin.services.s3.model.S3Exception: LifecycleConfiguration is not valid, expected CreateBucketConfiguration + + testBucket + } + private suspend fun getBucketWithPrefix(client: S3Client, prefix: String): String = withTimeout(60.seconds) { var testBucket = client.listBuckets() .buckets From 5f91b5d9acceb0f16a540ea63626c5443cef3a61 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 12:47:28 -0500 Subject: [PATCH 019/145] Simplify MD5 checksums interceptor --- .../customization/s3/ClientConfigIntegration.kt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index 97882e839d3..67251da8bb8 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -74,7 +74,7 @@ class ClientConfigIntegration : KotlinIntegration { val S3ExpressCredentialsProvider: ConfigProperty = ConfigProperty { name = "s3ExpressCredentialsProvider" symbol = buildSymbol { - name = "S3ExpressCredentialsProvider" + name = "SdkS3ExpressCredentialsProvider" nullable = false namespace = "aws.sdk.kotlin.services.s3.internal" } @@ -82,11 +82,19 @@ class ClientConfigIntegration : KotlinIntegration { Credentials provider to be used for making requests to S3 Express. """.trimIndent() + additionalImports = listOf( + buildSymbol { + name = "S3ExpressCredentialsProvider" + nullable = false + namespace = "aws.sdk.kotlin.services.s3.internal" // FIXME this should not be "internal" -- users will implement their own providers off this interface + } + ) + // FIXME Figure out why the default value is not being used properly. // propertyType = ConfigPropertyType.RequiredWithDefault("S3ExpressCredentialsProvider(this.credentialsProvider)") propertyType = ConfigPropertyType.Custom( render = { _, writer -> - writer.write("public val $name: S3ExpressCredentialsProvider = builder.$name ?: S3ExpressCredentialsProvider(this.credentialsProvider)") + writer.write("public val $name: S3ExpressCredentialsProvider = builder.$name ?: SdkS3ExpressCredentialsProvider(this.credentialsProvider)") }, renderBuilder = { prop, writer -> prop.documentation?.let(writer::dokka) From 78705487e3764ef8616c7be01b736dae39769aec Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 12:47:53 -0500 Subject: [PATCH 020/145] create S3ExpressCredentialsProvider interface --- .../s3/internal/S3ExpressCredentialsProvider.kt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt index 1aa7f3442b8..2102486b7dc 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt @@ -5,22 +5,23 @@ package aws.sdk.kotlin.services.s3.internal import aws.sdk.kotlin.runtime.auth.S3ExpressAttributes -import aws.sdk.kotlin.runtime.auth.credentials.internal.credentials -import aws.sdk.kotlin.services.s3.* -import aws.sdk.kotlin.services.s3.model.CreateSessionRequest import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvider import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.telemetry.logging.logger -import aws.smithy.kotlin.runtime.util.ExpiringValue import kotlin.coroutines.coroutineContext import aws.smithy.kotlin.runtime.io.SdkManagedBase -public class S3ExpressCredentialsProvider( - public val bootstrapCredentialsProvider: CredentialsProvider, -) : CloseableCredentialsProvider, SdkManagedBase() { +public interface S3ExpressCredentialsProvider: CloseableCredentialsProvider { + override suspend fun resolve(attributes: Attributes): Credentials + override fun close() +} + +public class SdkS3ExpressCredentialsProvider( + public val bootstrapCredentialsProvider: CredentialsProvider +) : S3ExpressCredentialsProvider, SdkManagedBase() { private val credentialsCache = S3ExpressCredentialsCache() override suspend fun resolve(attributes: Attributes): Credentials { From 012b4bed957867cf052b12b7a04ccb9568858b19 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 12:48:54 -0500 Subject: [PATCH 021/145] Simplify and add a test --- .../s3/internal/S3ExpressCredentialsCache.kt | 38 ++++++++++++------- .../internal/S3ExpressCredentialsCacheTest.kt | 29 ++++++++++++++ 2 files changed, 53 insertions(+), 14 deletions(-) create mode 100644 services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 6b0693d327d..6fc50c7b5df 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -14,15 +14,12 @@ import aws.smithy.kotlin.runtime.time.Instant import kotlinx.coroutines.* import kotlinx.coroutines.channels.* import kotlinx.coroutines.selects.* -import kotlin.coroutines.coroutineContext import aws.sdk.kotlin.services.s3.* -import aws.sdk.kotlin.services.s3.model.CreateSessionRequest import aws.sdk.kotlin.runtime.auth.credentials.internal.credentials +import aws.smithy.kotlin.runtime.io.use import kotlin.time.Duration.Companion.minutes import kotlin.coroutines.CoroutineContext import aws.smithy.kotlin.runtime.telemetry.logging.logger -import kotlin.time.Duration -import kotlin.time.* import aws.smithy.kotlin.runtime.time.until private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 @@ -43,7 +40,7 @@ public class S3ExpressCredentialsCache( } } - public suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru.get(key)?.value + public suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru.get(key)?.takeIf { it.isExpired(clock) }?.value ?: (createSessionCredentials(key).also { put(key, it) }).value public suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue): Unit { @@ -51,6 +48,17 @@ public class S3ExpressCredentialsCache( immediateRefreshChannel.send(Unit) } + private fun ExpiringValue.isExpired(clock: Clock): Boolean { + return clock.now().until(expiresAt).absoluteValue <= REFRESH_BUFFER + } + + /** + * Attempt to refresh the credentials in the cache. A refresh is initiated when: + * * a new set of credentials are added to the cache (immediate refresh) + * * the `nextRefresh` time has been reached, which is either `DEFAULT_REFRESH_PERIOD` or + * the soonest credentials expiration time (minus a buffer), whichever comes first. + * + */ private suspend fun refresh(): Unit { val logger = coroutineContext.logger() while (isActive) { @@ -63,14 +71,14 @@ public class S3ExpressCredentialsCache( lru.entries.forEach { (key, cachedValue) -> logger.trace { "Checking entry for ${key.bucket}" } println("Checking entry for ${key.bucket}") - nextRefresh = minOf(nextRefresh, cachedValue.expiresAt) + nextRefresh = minOf(nextRefresh, cachedValue.expiresAt - REFRESH_BUFFER) - if ((clock.now().until(cachedValue.expiresAt)).absoluteValue <= REFRESH_BUFFER) { + if (cachedValue.isExpired(clock)) { logger.trace { "Credentials for ${key.bucket} expire within the refresh buffer period, performing a refresh..." } println("Credentials for ${key.bucket} expire within the refresh buffer period, performing a refresh...") createSessionCredentials(key).also { - refreshedCredentials.put(key, it) - nextRefresh = minOf(nextRefresh, it.expiresAt) + refreshedCredentials[key] = it + nextRefresh = minOf(nextRefresh, it.expiresAt - REFRESH_BUFFER) } } } @@ -98,11 +106,13 @@ public class S3ExpressCredentialsCache( private suspend fun createSessionCredentials(key: S3ExpressCredentialsCacheKey): ExpiringValue { val logger = coroutineContext.logger() - val credentials = (key.client as S3Client).createSession( - CreateSessionRequest { - bucket = key.bucket - }, - ).credentials!! + val credentials = (key.client as S3Client) + // de-configure interceptors because this key.client is the user's S3 client, and we don't want to + // execute their custom interceptors during this internal createSession request + .withConfig { interceptors = mutableListOf() } + .use { + it.createSession { bucket = key.bucket }.credentials!! + } return ExpiringValue( credentials( diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt new file mode 100644 index 00000000000..8a7026efa78 --- /dev/null +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt @@ -0,0 +1,29 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.services.s3.internal + +import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals + +public class S3ExpressCredentialsCacheTest { + // val clock = ManualClock() + /** + * Different keys with the same bucket, client, and credentials should be considered equal + */ + @Test + fun testCacheKeyEquality() = runTest { + val bucket = "bucket" + val client = S3Client.builder().build() + val testCredentials = Credentials("accessKeyId", "secretAccessKey", "sessionToken") + + val key1 = S3ExpressCredentialsCacheKey(bucket, client, testCredentials) + val key2 = S3ExpressCredentialsCacheKey(bucket, client, testCredentials) + + assertEquals(key1, key2) + } +} From 9e43b1beeaeb1a8b63d5abedfad32c082a4a0a14 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 12:49:27 -0500 Subject: [PATCH 022/145] Add new dependency in order to use `AwsHttpSigner` --- aws-runtime/aws-http/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/aws-runtime/aws-http/build.gradle.kts b/aws-runtime/aws-http/build.gradle.kts index c91aa409876..21a560d31a8 100644 --- a/aws-runtime/aws-http/build.gradle.kts +++ b/aws-runtime/aws-http/build.gradle.kts @@ -15,6 +15,7 @@ kotlin { api(project(":aws-runtime:aws-endpoint")) api(libs.smithy.kotlin.aws.signing.common) api(libs.smithy.kotlin.http.client) + api(libs.smithy.kotlin.http.auth.aws) } } From 80dfb81b25622460bf1dba77904a48019c0a7fea Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 13:23:36 -0500 Subject: [PATCH 023/145] cleanup --- .../kotlin/services/s3/internal/S3ExpressCredentialsCache.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 6fc50c7b5df..5a5cbc72632 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -57,7 +57,6 @@ public class S3ExpressCredentialsCache( * * a new set of credentials are added to the cache (immediate refresh) * * the `nextRefresh` time has been reached, which is either `DEFAULT_REFRESH_PERIOD` or * the soonest credentials expiration time (minus a buffer), whichever comes first. - * */ private suspend fun refresh(): Unit { val logger = coroutineContext.logger() @@ -106,6 +105,7 @@ public class S3ExpressCredentialsCache( private suspend fun createSessionCredentials(key: S3ExpressCredentialsCacheKey): ExpiringValue { val logger = coroutineContext.logger() + // FIXME Consider creating a brand new client using the bootstrapped key.credentials instead of abusing the user's client val credentials = (key.client as S3Client) // de-configure interceptors because this key.client is the user's S3 client, and we don't want to // execute their custom interceptors during this internal createSession request From 7f25eed9bb9ae6f005205b149b4a3c3e9c2153df Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 16:27:44 -0500 Subject: [PATCH 024/145] CRT signer working --- .../src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt index c5b46bae52f..b4fe51cedbc 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt @@ -31,7 +31,7 @@ public class SigV4S3ExpressAuthScheme( AwsHttpSigner.Config().apply { signer = awsSigner service = serviceName - algorithm = AwsSigningAlgorithm.SIGV4_S3EXPRESS + algorithm = AwsSigningAlgorithm.SIGV4 // Note: There is no new signing algorithm for S3 Express } )) From f00ca0a9b4eedfb5e396760b7bb1f76e4225f651 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 7 Feb 2024 17:07:08 -0500 Subject: [PATCH 025/145] docs --- .../codegen/customization/s3/express/S3ExpressIntegration.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 81a0fda6590..ad4decc438c 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -31,10 +31,10 @@ import software.amazon.smithy.model.traits.HttpChecksumRequiredTrait import software.amazon.smithy.model.traits.HttpHeaderTrait /** - * An integration which sets up multiple code-generation aspects for S3 Express. + * An integration which handles codegen for S3 Express, such as: * 1. Configure auth scheme (auth scheme ID, auth option, identity provider for auth scheme) * 2. Add S3Client and Bucket to execution context (required for the S3 Express credentials cache key) - * 3. Override all checksums to use CRC32 instead + * 3. Override checksums to use CRC32 instead of MD5 * 4. Disable all checksums for s3:UploadPart */ class S3ExpressIntegration : KotlinIntegration { From e6fb9f56aca05727baad3631cae4cf7f15623ab4 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 8 Feb 2024 09:45:17 -0500 Subject: [PATCH 026/145] Relocate S3 Express AuthScheme ID to aws-sdk-kotlin --- .../runtime/auth/SigV4S3ExpressAuthScheme.kt | 6 +++-- .../SigV4S3ExpressAuthSchemeIntegration.kt | 4 ++-- .../s3/express/SigV4S3ExpressTrait.kt | 24 ------------------- 3 files changed, 6 insertions(+), 28 deletions(-) delete mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressTrait.kt diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt index b4fe51cedbc..52715754bc8 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt @@ -21,6 +21,8 @@ import aws.smithy.kotlin.runtime.http.auth.AuthScheme import aws.smithy.kotlin.runtime.http.auth.AwsHttpSigner import aws.smithy.kotlin.runtime.http.auth.HttpSigner +private val S3_EXPRESS_AUTH_SCHEME_ID = AuthSchemeId("aws.auth#sigv4s3express") + /** * HTTP auth scheme for S3 Express One Zone authentication */ @@ -35,7 +37,7 @@ public class SigV4S3ExpressAuthScheme( } )) - override val schemeId: AuthSchemeId = AuthSchemeId.AwsSigV4S3Express + override val schemeId: AuthSchemeId = S3_EXPRESS_AUTH_SCHEME_ID override val signer: HttpSigner = S3ExpressHttpSigner(awsHttpSigner) } @@ -64,7 +66,7 @@ public fun sigV4S3Express( } else { emptyAttributes() } - return AuthOption(AuthSchemeId.AwsSigV4S3Express, attrs) + return AuthOption(S3_EXPRESS_AUTH_SCHEME_ID, attrs) } // FIXME Copied from SigV4AuthScheme.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index b517a81b46c..7d42a9d70ca 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -60,10 +60,10 @@ private object SigV4S3ExpressEndpointCustomization : EndpointCustomization { } open class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { - override val authSchemeId: ShapeId = SigV4S3ExpressTrait.ID + override val authSchemeId: ShapeId = ShapeId.from("aws.auth#sigv4s3express") override val authSchemeIdSymbol: Symbol = buildSymbol { - name = "AuthSchemeId.AwsSigV4S3Express" + name = "AuthSchemeId(\"aws.auth#sigv4s3express\")" val ref = RuntimeTypes.Auth.Identity.AuthSchemeId objectRef = ref namespace = ref.namespace diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressTrait.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressTrait.kt deleted file mode 100644 index 2b7e11de406..00000000000 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressTrait.kt +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.codegen.customization.s3.express - -import software.amazon.smithy.model.node.Node -import software.amazon.smithy.model.shapes.ShapeId -import software.amazon.smithy.model.traits.AbstractTrait - -/** - * Synthetic trait representing the `sigv4-s3express` auth scheme. - */ -public class SigV4S3ExpressTrait : AbstractTrait(ID, Node.objectNode()) { - public companion object { - val ID = ShapeId.from("com.amazonaws.s3#sigv4express") - // FIXME What shape ID should be used for sigv4-s3express? It's not in Smithy... - // Go v2 uses this. https://github.com/aws/aws-sdk-go-v2/blob/4ba37053fff9055b216a303ab591f6fa5e4c80c1/service/s3/endpoint_auth_resolver.go#L29C1-L29C1 - } - - override fun createNode(): Node = Node.objectNode() - - override fun isSynthetic(): Boolean = true -} From 1cb4caa9a3ca00e05e69f0a60bee8c76fa02058d Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 8 Feb 2024 10:06:22 -0500 Subject: [PATCH 027/145] Remove log --- .../kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt index 2102486b7dc..6a58b9cd9e5 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt @@ -32,7 +32,7 @@ public class SdkS3ExpressCredentialsProvider( val key = S3ExpressCredentialsCacheKey(bucket, client, bootstrapCredentialsProvider.resolve(attributes)) - return credentialsCache.get(key).also { logger.trace { "Got credentials $it from cache" }} + return credentialsCache.get(key) } override fun close() { From bbe3e35981a6dc3def2936d13b91998d5306d7e7 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 8 Feb 2024 10:14:07 -0500 Subject: [PATCH 028/145] Cleanup --- .../services/s3/internal/S3ExpressCredentialsCacheTest.kt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt index 8a7026efa78..345f18bf889 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt @@ -11,16 +11,14 @@ import kotlin.test.Test import kotlin.test.assertEquals public class S3ExpressCredentialsCacheTest { - // val clock = ManualClock() - /** - * Different keys with the same bucket, client, and credentials should be considered equal - */ @Test fun testCacheKeyEquality() = runTest { + val bucket = "bucket" val client = S3Client.builder().build() val testCredentials = Credentials("accessKeyId", "secretAccessKey", "sessionToken") + // Different keys with the same bucket, client, and credentials should be considered equal val key1 = S3ExpressCredentialsCacheKey(bucket, client, testCredentials) val key2 = S3ExpressCredentialsCacheKey(bucket, client, testCredentials) From e6ff901dd2b0b1cdea558962e6d2aafedca7b9b8 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 8 Feb 2024 10:25:30 -0500 Subject: [PATCH 029/145] Revert visibility of credentials() function --- aws-runtime/aws-config/api/aws-config.api | 5 ----- .../runtime/auth/credentials/internal/CredentialsExt.kt | 2 +- .../kotlin/services/s3/internal/S3ExpressCredentialsCache.kt | 3 +-- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index e662d3ab747..b7a3996b402 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -225,11 +225,6 @@ public final class aws/sdk/kotlin/runtime/auth/credentials/SystemPropertyCredent public fun resolve (Laws/smithy/kotlin/runtime/collections/Attributes;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; } -public final class aws/sdk/kotlin/runtime/auth/credentials/internal/CredentialsExtKt { - public static final fun credentials (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/time/Instant;Ljava/lang/String;Ljava/lang/String;)Laws/smithy/kotlin/runtime/auth/awscredentials/Credentials; - public static synthetic fun credentials$default (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/time/Instant;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Laws/smithy/kotlin/runtime/auth/awscredentials/Credentials; -} - public final class aws/sdk/kotlin/runtime/auth/credentials/internal/ManagedBearerTokenProviderKt { public static final fun manage (Laws/smithy/kotlin/runtime/http/auth/CloseableBearerTokenProvider;)Laws/smithy/kotlin/runtime/http/auth/BearerTokenProvider; } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/internal/CredentialsExt.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/internal/CredentialsExt.kt index b98822f457a..3f40a0e584d 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/internal/CredentialsExt.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/credentials/internal/CredentialsExt.kt @@ -12,7 +12,7 @@ import aws.smithy.kotlin.runtime.collections.setIfValueNotNull import aws.smithy.kotlin.runtime.identity.IdentityAttributes import aws.smithy.kotlin.runtime.time.Instant -public fun credentials( // FIXME can we make this public? if not, how should I create [Credentials] from within the s3 package? +internal fun credentials( accessKeyId: String, secretAccessKey: String, sessionToken: String? = null, diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 5a5cbc72632..4b849d0b71d 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -15,7 +15,6 @@ import kotlinx.coroutines.* import kotlinx.coroutines.channels.* import kotlinx.coroutines.selects.* import aws.sdk.kotlin.services.s3.* -import aws.sdk.kotlin.runtime.auth.credentials.internal.credentials import aws.smithy.kotlin.runtime.io.use import kotlin.time.Duration.Companion.minutes import kotlin.coroutines.CoroutineContext @@ -115,7 +114,7 @@ public class S3ExpressCredentialsCache( } return ExpiringValue( - credentials( + Credentials.invoke( accessKeyId = credentials.accessKeyId, secretAccessKey = credentials.secretAccessKey, sessionToken = credentials.sessionToken, From 228333fbedb2410a1dc0aaf42c79037f26080623 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 8 Feb 2024 10:32:22 -0500 Subject: [PATCH 030/145] ktlintFormat --- .../runtime/auth/SigV4S3ExpressAuthScheme.kt | 16 +++++++------ .../runtime/http/S3ExpressHttpSigner.kt | 10 +++++--- .../s3/ClientConfigIntegration.kt | 2 +- .../s3/express/S3ExpressIntegration.kt | 10 ++++---- .../s3/internal/S3ExpressCredentialsCache.kt | 24 +++++++++---------- .../internal/S3ExpressCredentialsProvider.kt | 8 +++---- .../internal/S3ExpressCredentialsCacheTest.kt | 1 - services/s3/e2eTest/src/S3ExpressTest.kt | 10 ++------ services/s3/e2eTest/src/S3TestUtils.kt | 6 ++--- 9 files changed, 42 insertions(+), 45 deletions(-) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt index 52715754bc8..70108e9f0e8 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt @@ -29,13 +29,15 @@ private val S3_EXPRESS_AUTH_SCHEME_ID = AuthSchemeId("aws.auth#sigv4s3express") public class SigV4S3ExpressAuthScheme( awsHttpSigner: AwsHttpSigner, ) : AuthScheme { - public constructor(awsSigner: AwsSigner, serviceName: String? = null) : this(AwsHttpSigner( - AwsHttpSigner.Config().apply { - signer = awsSigner - service = serviceName - algorithm = AwsSigningAlgorithm.SIGV4 // Note: There is no new signing algorithm for S3 Express - } - )) + public constructor(awsSigner: AwsSigner, serviceName: String? = null) : this( + AwsHttpSigner( + AwsHttpSigner.Config().apply { + signer = awsSigner + service = serviceName + algorithm = AwsSigningAlgorithm.SIGV4 // Note: There is no new signing algorithm for S3 Express + }, + ), + ) override val schemeId: AuthSchemeId = S3_EXPRESS_AUTH_SCHEME_ID override val signer: HttpSigner = S3ExpressHttpSigner(awsHttpSigner) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt index ac6af0d4811..8620a819daa 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt @@ -1,3 +1,7 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ package aws.sdk.kotlin.runtime.http import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials @@ -18,8 +22,8 @@ private const val SESSION_TOKEN_HEADER = "X-Amz-Security-Token" * @param awsHttpSigner An instance of [AwsHttpSigner] */ public class S3ExpressHttpSigner( - public val awsHttpSigner: AwsHttpSigner -): HttpSigner { + public val awsHttpSigner: AwsHttpSigner, +) : HttpSigner { /** * Sign the request, adding `X-Amz-S3Session-Token` header and removing `X-Amz-Security-Token` header. */ @@ -40,4 +44,4 @@ public class S3ExpressHttpSigner( // 4. remove session token header signingRequest.httpRequest.headers.remove(SESSION_TOKEN_HEADER) } -} \ No newline at end of file +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index 67251da8bb8..7f829b7ec63 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -87,7 +87,7 @@ class ClientConfigIntegration : KotlinIntegration { name = "S3ExpressCredentialsProvider" nullable = false namespace = "aws.sdk.kotlin.services.s3.internal" // FIXME this should not be "internal" -- users will implement their own providers off this interface - } + }, ) // FIXME Figure out why the default value is not being used properly. diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index ad4decc438c..960a4fad525 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -63,11 +63,11 @@ class S3ExpressIntegration : KotlinIntegration { override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = resolved + listOf( - AddClientToExecutionContext, - AddBucketToExecutionContext, - UseCrc32Checksum, - UploadPartDisableChecksum, - ) + AddClientToExecutionContext, + AddBucketToExecutionContext, + UseCrc32Checksum, + UploadPartDisableChecksum, + ) private val AddClientToExecutionContext = object : ProtocolMiddleware { override val name: String = "AddClientToExecutionContext" diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 4b849d0b71d..3448eb23591 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -4,22 +4,22 @@ */ package aws.sdk.kotlin.services.s3.internal +import aws.sdk.kotlin.services.s3.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.collections.LruCache -import aws.smithy.kotlin.runtime.time.Clock -import aws.smithy.kotlin.runtime.util.ExpiringValue import aws.smithy.kotlin.runtime.io.Closeable +import aws.smithy.kotlin.runtime.io.use +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import aws.smithy.kotlin.runtime.time.Clock import aws.smithy.kotlin.runtime.time.Instant +import aws.smithy.kotlin.runtime.time.until +import aws.smithy.kotlin.runtime.util.ExpiringValue import kotlinx.coroutines.* import kotlinx.coroutines.channels.* import kotlinx.coroutines.selects.* -import aws.sdk.kotlin.services.s3.* -import aws.smithy.kotlin.runtime.io.use -import kotlin.time.Duration.Companion.minutes import kotlin.coroutines.CoroutineContext -import aws.smithy.kotlin.runtime.telemetry.logging.logger -import aws.smithy.kotlin.runtime.time.until +import kotlin.time.Duration.Companion.minutes private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 private val REFRESH_BUFFER = 1.minutes @@ -42,14 +42,12 @@ public class S3ExpressCredentialsCache( public suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru.get(key)?.takeIf { it.isExpired(clock) }?.value ?: (createSessionCredentials(key).also { put(key, it) }).value - public suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue): Unit { + public suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue) { lru.put(key, value) immediateRefreshChannel.send(Unit) } - private fun ExpiringValue.isExpired(clock: Clock): Boolean { - return clock.now().until(expiresAt).absoluteValue <= REFRESH_BUFFER - } + private fun ExpiringValue.isExpired(clock: Clock): Boolean = clock.now().until(expiresAt).absoluteValue <= REFRESH_BUFFER /** * Attempt to refresh the credentials in the cache. A refresh is initiated when: @@ -57,7 +55,7 @@ public class S3ExpressCredentialsCache( * * the `nextRefresh` time has been reached, which is either `DEFAULT_REFRESH_PERIOD` or * the soonest credentials expiration time (minus a buffer), whichever comes first. */ - private suspend fun refresh(): Unit { + private suspend fun refresh() { val logger = coroutineContext.logger() while (isActive) { logger.trace { "Looping..." } @@ -125,7 +123,7 @@ public class S3ExpressCredentialsCache( ).also { logger.debug { "got credentials ${it.value}" } } } - override fun close(): Unit { + override fun close() { coroutineContext.cancel(null) immediateRefreshChannel.close() } diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt index 6a58b9cd9e5..44c0d0aa041 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt @@ -5,22 +5,22 @@ package aws.sdk.kotlin.services.s3.internal import aws.sdk.kotlin.runtime.auth.S3ExpressAttributes +import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvider import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider -import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvider import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.collections.get +import aws.smithy.kotlin.runtime.io.SdkManagedBase import aws.smithy.kotlin.runtime.telemetry.logging.logger import kotlin.coroutines.coroutineContext -import aws.smithy.kotlin.runtime.io.SdkManagedBase -public interface S3ExpressCredentialsProvider: CloseableCredentialsProvider { +public interface S3ExpressCredentialsProvider : CloseableCredentialsProvider { override suspend fun resolve(attributes: Attributes): Credentials override fun close() } public class SdkS3ExpressCredentialsProvider( - public val bootstrapCredentialsProvider: CredentialsProvider + public val bootstrapCredentialsProvider: CredentialsProvider, ) : S3ExpressCredentialsProvider, SdkManagedBase() { private val credentialsCache = S3ExpressCredentialsCache() diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt index 345f18bf889..840dee9c5aa 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt @@ -13,7 +13,6 @@ import kotlin.test.assertEquals public class S3ExpressCredentialsCacheTest { @Test fun testCacheKeyEquality() = runTest { - val bucket = "bucket" val client = S3Client.builder().build() val testCredentials = Credentials("accessKeyId", "secretAccessKey", "sessionToken") diff --git a/services/s3/e2eTest/src/S3ExpressTest.kt b/services/s3/e2eTest/src/S3ExpressTest.kt index 12f50f19cb9..bd5097cc6a4 100644 --- a/services/s3/e2eTest/src/S3ExpressTest.kt +++ b/services/s3/e2eTest/src/S3ExpressTest.kt @@ -5,15 +5,10 @@ package aws.sdk.kotlin.e2etest import aws.sdk.kotlin.services.s3.S3Client -import aws.sdk.kotlin.services.s3.internal.S3ExpressCredentialsProvider -import aws.sdk.kotlin.services.s3.internal.SdkS3ExpressCredentialsProvider import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.putObject import aws.sdk.kotlin.services.s3.withConfig -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext -import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.decodeToString import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor @@ -29,7 +24,6 @@ import kotlin.test.* * Tests for S3 Express operations */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) - class S3ExpressTest { private val client = S3Client { region = S3TestUtils.DEFAULT_REGION @@ -111,11 +105,11 @@ class S3ExpressTest { } } - private class CRC32ChecksumValidatingInterceptor: HttpInterceptor { + private class CRC32ChecksumValidatingInterceptor : HttpInterceptor { override fun readAfterSigning(context: ProtocolRequestInterceptorContext) { val headers = context.protocolRequest.headers assertTrue(headers.contains("x-amz-checksum-crc32"), "Failed to find x-amz-checksum-crc32 header") assertFalse(headers.contains("Content-MD5"), "Unexpectedly found Content-MD5 header") } } -} \ No newline at end of file +} diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 2dbf720492c..06cf750660b 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -45,9 +45,9 @@ object S3TestUtils { println("Creating S3 bucket: $testBucket") - val availabilityZone = testBucket // s3-test-bucket-UUID--use1-az4--x-s3 - .removeSuffix(S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX) // s3-test-bucket-UUID--use1-az4 - .substringAfterLast("--") // use1-az4 + val availabilityZone = testBucket // s3-test-bucket-UUID--use1-az4--x-s3 + .removeSuffix(S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX) // s3-test-bucket-UUID--use1-az4 + .substringAfterLast("--") // use1-az4 client.createBucket { bucket = testBucket From 50ee25da569b4138fa88d17a21c4cfa1a2454d9d Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 8 Feb 2024 12:39:51 -0500 Subject: [PATCH 031/145] cleanup --- aws-runtime/aws-http/api/aws-http.api | 96 +++++++++---------- .../s3/ClientConfigIntegration.kt | 2 - .../SigV4S3ExpressAuthSchemeIntegration.kt | 19 +--- 3 files changed, 51 insertions(+), 66 deletions(-) diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index b716520c2a4..88a9dfd857f 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -103,6 +103,12 @@ public final class aws/sdk/kotlin/runtime/http/OsMetadata { public fun toString ()Ljava/lang/String; } +public final class aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner : aws/smithy/kotlin/runtime/http/auth/HttpSigner { + public fun (Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner;)V + public final fun getAwsHttpSigner ()Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner; + public fun sign (Laws/smithy/kotlin/runtime/http/auth/SignHttpRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; +} + public final class aws/sdk/kotlin/runtime/http/SdkMetadata { public fun (Ljava/lang/String;Ljava/lang/String;)V public final fun component1 ()Ljava/lang/String; @@ -162,8 +168,11 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/AwsSpanInterceptor : public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } -public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAlgorithmInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { +public final class aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { public fun ()V + public fun (Ljava/lang/String;)V + public synthetic fun (Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getChecksumAlgorithmHeaderName ()Ljava/lang/String; public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public fun modifyBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; @@ -185,53 +194,8 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAl public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } -public final class aws/sdk/kotlin/runtime/http/middleware/AwsRetryHeaderMiddleware : aws/smithy/kotlin/runtime/http/operation/MutateMiddleware { - public fun ()V - public fun handle (Laws/smithy/kotlin/runtime/http/operation/OperationRequest;Laws/smithy/kotlin/runtime/io/Handler;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public synthetic fun handle (Ljava/lang/Object;Laws/smithy/kotlin/runtime/io/Handler;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun install (Laws/smithy/kotlin/runtime/http/operation/SdkHttpOperation;)V -} - -public final class aws/sdk/kotlin/runtime/http/middleware/RecursionDetection : aws/smithy/kotlin/runtime/http/operation/ModifyRequestMiddleware { - public fun ()V - public fun (Laws/smithy/kotlin/runtime/util/EnvironmentProvider;)V - public synthetic fun (Laws/smithy/kotlin/runtime/util/EnvironmentProvider;ILkotlin/jvm/internal/DefaultConstructorMarker;)V - public fun install (Laws/smithy/kotlin/runtime/http/operation/SdkHttpOperation;)V - public fun modifyRequest (Laws/smithy/kotlin/runtime/http/operation/OperationRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public synthetic fun modifyRequest (Ljava/lang/Object;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; -} - -public final class aws/sdk/kotlin/runtime/http/middleware/UserAgent : aws/smithy/kotlin/runtime/http/operation/ModifyRequestMiddleware { - public fun (Laws/sdk/kotlin/runtime/http/AwsUserAgentMetadata;)V - public fun install (Laws/smithy/kotlin/runtime/http/operation/SdkHttpOperation;)V - public fun modifyRequest (Laws/smithy/kotlin/runtime/http/operation/OperationRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public synthetic fun modifyRequest (Ljava/lang/Object;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; -} - -public final class aws/sdk/kotlin/runtime/http/operation/ConfigMetadata : aws/sdk/kotlin/runtime/http/operation/TypedUserAgentMetadata { - public fun (Ljava/lang/String;Ljava/lang/String;)V - public final fun component1 ()Ljava/lang/String; - public final fun component2 ()Ljava/lang/String; - public final fun copy (Ljava/lang/String;Ljava/lang/String;)Laws/sdk/kotlin/runtime/http/operation/ConfigMetadata; - public static synthetic fun copy$default (Laws/sdk/kotlin/runtime/http/operation/ConfigMetadata;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Laws/sdk/kotlin/runtime/http/operation/ConfigMetadata; - public fun equals (Ljava/lang/Object;)Z - public final fun getName ()Ljava/lang/String; - public final fun getValue ()Ljava/lang/String; - public fun hashCode ()I - public fun toString ()Ljava/lang/String; -} - -public final class aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner : aws/smithy/kotlin/runtime/http/auth/HttpSigner { - public fun (Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner;)V - public final fun getAwsHttpSigner ()Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner; - public fun sign (Laws/smithy/kotlin/runtime/http/auth/SignHttpRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; -} - -public final class aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { +public final class aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { public fun ()V - public fun (Ljava/lang/String;)V - public synthetic fun (Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V - public final fun getChecksumAlgorithmHeaderName ()Ljava/lang/String; public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public fun modifyBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; @@ -253,7 +217,7 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32Checks public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } -public final class aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { +public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAlgorithmInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { public fun ()V public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; @@ -276,6 +240,42 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChec public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } +public final class aws/sdk/kotlin/runtime/http/middleware/AwsRetryHeaderMiddleware : aws/smithy/kotlin/runtime/http/operation/MutateMiddleware { + public fun ()V + public fun handle (Laws/smithy/kotlin/runtime/http/operation/OperationRequest;Laws/smithy/kotlin/runtime/io/Handler;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public synthetic fun handle (Ljava/lang/Object;Laws/smithy/kotlin/runtime/io/Handler;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public fun install (Laws/smithy/kotlin/runtime/http/operation/SdkHttpOperation;)V +} + +public final class aws/sdk/kotlin/runtime/http/middleware/RecursionDetection : aws/smithy/kotlin/runtime/http/operation/ModifyRequestMiddleware { + public fun ()V + public fun (Laws/smithy/kotlin/runtime/util/EnvironmentProvider;)V + public synthetic fun (Laws/smithy/kotlin/runtime/util/EnvironmentProvider;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun install (Laws/smithy/kotlin/runtime/http/operation/SdkHttpOperation;)V + public fun modifyRequest (Laws/smithy/kotlin/runtime/http/operation/OperationRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public synthetic fun modifyRequest (Ljava/lang/Object;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; +} + +public final class aws/sdk/kotlin/runtime/http/middleware/UserAgent : aws/smithy/kotlin/runtime/http/operation/ModifyRequestMiddleware { + public fun (Laws/sdk/kotlin/runtime/http/AwsUserAgentMetadata;)V + public fun install (Laws/smithy/kotlin/runtime/http/operation/SdkHttpOperation;)V + public fun modifyRequest (Laws/smithy/kotlin/runtime/http/operation/OperationRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public synthetic fun modifyRequest (Ljava/lang/Object;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; +} + +public final class aws/sdk/kotlin/runtime/http/operation/ConfigMetadata : aws/sdk/kotlin/runtime/http/operation/TypedUserAgentMetadata { + public fun (Ljava/lang/String;Ljava/lang/String;)V + public final fun component1 ()Ljava/lang/String; + public final fun component2 ()Ljava/lang/String; + public final fun copy (Ljava/lang/String;Ljava/lang/String;)Laws/sdk/kotlin/runtime/http/operation/ConfigMetadata; + public static synthetic fun copy$default (Laws/sdk/kotlin/runtime/http/operation/ConfigMetadata;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Laws/sdk/kotlin/runtime/http/operation/ConfigMetadata; + public fun equals (Ljava/lang/Object;)Z + public final fun getName ()Ljava/lang/String; + public final fun getValue ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadata { public fun ()V public fun (Ljava/util/Map;Ljava/util/List;)V diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index 7f829b7ec63..8b9b0334118 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -90,8 +90,6 @@ class ClientConfigIntegration : KotlinIntegration { }, ) - // FIXME Figure out why the default value is not being used properly. -// propertyType = ConfigPropertyType.RequiredWithDefault("S3ExpressCredentialsProvider(this.credentialsProvider)") propertyType = ConfigPropertyType.Custom( render = { _, writer -> writer.write("public val $name: S3ExpressCredentialsProvider = builder.$name ?: SdkS3ExpressCredentialsProvider(this.credentialsProvider)") diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index 7d42a9d70ca..d91ba06e626 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -27,7 +27,6 @@ import software.amazon.smithy.kotlin.codegen.rendering.protocol.* import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.node.Node -import software.amazon.smithy.model.node.ObjectNode import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ServiceShape import software.amazon.smithy.model.shapes.ShapeId @@ -93,19 +92,13 @@ open class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { } } -private fun String.toAuthOptionFactoryFn(): Symbol? = - when (this) { - "sigv4-s3express" -> AwsRuntimeTypes.Config.Auth.sigV4S3Express - else -> null - } - private fun renderAuthSchemes(writer: KotlinWriter, authSchemes: Expression, expressionRenderer: ExpressionRenderer) { writer.writeInline("#T to ", RuntimeTypes.SmithyClient.Endpoints.SigningContextAttributeKey) writer.withBlock("listOf(", ")") { authSchemes.toNode().expectArrayNode().forEach { val scheme = it.expectObjectNode() val schemeName = scheme.expectStringMember("name").value - val authFactoryFn = schemeName.toAuthOptionFactoryFn() ?: return@forEach + val authFactoryFn = if (schemeName == "sigv4-s3express") AwsRuntimeTypes.Config.Auth.sigV4S3Express else return@forEach withBlock("#T(", "),", authFactoryFn) { // we delegate back to the expression visitor for each of these fields because it's possible to @@ -117,9 +110,8 @@ private fun renderAuthSchemes(writer: KotlinWriter, authSchemes: Expression, exp writeInline("disableDoubleUriEncode = ") renderOrElse(expressionRenderer, scheme.getBooleanMember("disableDoubleEncoding"), "false") - when (schemeName) { - "sigv4-s3express" -> renderSigV4ExpressFields(writer, scheme, expressionRenderer) - } + writer.writeInline("signingRegion = ") + writer.renderOrElse(expressionRenderer, scheme.getStringMember("signingRegion"), "null") } } } @@ -137,8 +129,3 @@ private fun KotlinWriter.renderOrElse( } write(",") } - -private fun renderSigV4ExpressFields(writer: KotlinWriter, scheme: ObjectNode, expressionRenderer: ExpressionRenderer) { - writer.writeInline("signingRegion = ") - writer.renderOrElse(expressionRenderer, scheme.getStringMember("signingRegion"), "null") -} From 777a52d2f8cf81234ddd8df4f6ccc6234f6bbcc4 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 8 Feb 2024 12:43:02 -0500 Subject: [PATCH 032/145] split S3ExpressCredentialsProvider interface from implementation --- .../customization/s3/ClientConfigIntegration.kt | 2 +- .../services/s3/S3ExpressCredentialsProvider.kt | 16 ++++++++++++++++ ...der.kt => SdkS3ExpressCredentialsProvider.kt} | 9 ++------- 3 files changed, 19 insertions(+), 8 deletions(-) create mode 100644 services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt rename services/s3/common/src/aws/sdk/kotlin/services/s3/internal/{S3ExpressCredentialsProvider.kt => SdkS3ExpressCredentialsProvider.kt} (83%) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index 8b9b0334118..5729b64e2ca 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -86,7 +86,7 @@ class ClientConfigIntegration : KotlinIntegration { buildSymbol { name = "S3ExpressCredentialsProvider" nullable = false - namespace = "aws.sdk.kotlin.services.s3.internal" // FIXME this should not be "internal" -- users will implement their own providers off this interface + namespace = "aws.sdk.kotlin.services.s3" }, ) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt new file mode 100644 index 00000000000..1f843d7596b --- /dev/null +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt @@ -0,0 +1,16 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.services.s3 + +import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvider +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.collections.Attributes + +public interface S3ExpressCredentialsProvider : CloseableCredentialsProvider { + override suspend fun resolve(attributes: Attributes): Credentials + override fun close() +} + + diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt similarity index 83% rename from services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt index 44c0d0aa041..959e314b7a5 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt @@ -5,7 +5,7 @@ package aws.sdk.kotlin.services.s3.internal import aws.sdk.kotlin.runtime.auth.S3ExpressAttributes -import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvider +import aws.sdk.kotlin.services.s3.S3ExpressCredentialsProvider import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider import aws.smithy.kotlin.runtime.collections.Attributes @@ -14,11 +14,6 @@ import aws.smithy.kotlin.runtime.io.SdkManagedBase import aws.smithy.kotlin.runtime.telemetry.logging.logger import kotlin.coroutines.coroutineContext -public interface S3ExpressCredentialsProvider : CloseableCredentialsProvider { - override suspend fun resolve(attributes: Attributes): Credentials - override fun close() -} - public class SdkS3ExpressCredentialsProvider( public val bootstrapCredentialsProvider: CredentialsProvider, ) : S3ExpressCredentialsProvider, SdkManagedBase() { @@ -38,4 +33,4 @@ public class SdkS3ExpressCredentialsProvider( override fun close() { credentialsCache.close() } -} +} \ No newline at end of file From 8440be1a01f18cbc31a4cd47952a857fd43b7c25 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 9 Feb 2024 09:13:32 -0500 Subject: [PATCH 033/145] correct expiration logic --- .../kotlin/services/s3/internal/S3ExpressCredentialsCache.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 3448eb23591..c217397417f 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -39,7 +39,7 @@ public class S3ExpressCredentialsCache( } } - public suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru.get(key)?.takeIf { it.isExpired(clock) }?.value + public suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru.get(key)?.takeIf { !it.isExpired(clock) }?.value ?: (createSessionCredentials(key).also { put(key, it) }).value public suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue) { From fd1ad4af9a33afb94e003b9ae14a5d95c5985a90 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 9 Feb 2024 13:18:48 -0500 Subject: [PATCH 034/145] Add S3 Express benchmark (temporarily commented-out other benchmarks so I can run this in EC2) --- .../benchmarks/service/BenchmarkHarness.kt | 15 +-- .../service/definitions/S3ExpressBenchmark.kt | 97 +++++++++++++++++++ .../telemetry/BenchmarkTelemetryProvider.kt | 10 +- 3 files changed, 110 insertions(+), 12 deletions(-) create mode 100644 tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt diff --git a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt index 4f523a93f2d..25189591afe 100644 --- a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt +++ b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt @@ -15,13 +15,14 @@ val DEFAULT_WARMUP_TIME = 5.seconds val DEFAULT_ITERATION_TIME = 15.seconds private val benchmarks = setOf( - S3Benchmark(), - SnsBenchmark(), - StsBenchmark(), - CloudwatchBenchmark(), - CloudwatchEventsBenchmark(), - DynamoDbBenchmark(), - PinpointBenchmark(), +// S3Benchmark(), +// SnsBenchmark(), +// StsBenchmark(), +// CloudwatchBenchmark(), +// CloudwatchEventsBenchmark(), +// DynamoDbBenchmark(), +// PinpointBenchmark(), + S3ExpressBenchmark() ).map { @Suppress("UNCHECKED_CAST") it as ServiceBenchmark diff --git a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt new file mode 100644 index 00000000000..2ae21d7ce65 --- /dev/null +++ b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt @@ -0,0 +1,97 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.benchmarks.service.definitions + +import aws.sdk.kotlin.benchmarks.service.Common +import aws.sdk.kotlin.services.s3.* +import aws.sdk.kotlin.services.s3.model.* +import aws.smithy.kotlin.runtime.ExperimentalApi +import aws.smithy.kotlin.runtime.content.ByteStream + +/** + * Benchmarks for S3 Express One Zone. + * Note: This benchmark must be run from an EC2 host in the same AZ as the bucket (usw2-az1). + */ +class S3ExpressBenchmark : ServiceBenchmark { + private val regionAz = "usw2-az1" + private val bucketName = Common.random("sdk-benchmark-bucket-") + .substring(0 until 47) + // truncate to prevent "bucket name too long" errors + "--$regionAz--x-s3" + + private val KEY = "64kb-object" + private val CONTENTS = "a".repeat(65536) // 64MB + + @OptIn(ExperimentalApi::class) + override suspend fun client() = S3Client.fromEnvironment { + retryStrategy = Common.noRetries + telemetryProvider = Common.telemetryProvider + httpClient { + telemetryProvider = Common.telemetryProvider + } + } + + override suspend fun setup(client: S3Client) { + client.createBucket { + bucket = bucketName + createBucketConfiguration { + location { + type = LocationType.AvailabilityZone + this.name = regionAz + } + bucket { + type = BucketType.Directory + dataRedundancy = DataRedundancy.SingleAvailabilityZone + } + } + } + } + + override val operations get() = listOf(putObjectBenchmark, getObjectBenchmark) + + override suspend fun tearDown(client: S3Client) { + client.deleteBucket { bucket = bucketName } + } + + private val putObjectBenchmark = object : AbstractOperationBenchmark("s3express:PutObject") { + override suspend fun transact(client: S3Client) { + client.putObject { + bucket = bucketName + key = KEY + body = ByteStream.fromString(CONTENTS) + } + } + + override suspend fun tearDown(client: S3Client) { + client.deleteObject { + bucket = bucketName + key = KEY + } + } + } + + private val getObjectBenchmark = object : AbstractOperationBenchmark("s3express:GetObject") { + override suspend fun setup(client: S3Client) { + client.putObject { + bucket = bucketName + key = KEY + body = ByteStream.fromString(CONTENTS) + } + } + + override suspend fun transact(client: S3Client) { + client.getObject(GetObjectRequest { + bucket = bucketName + key = KEY + }) { } + } + + override suspend fun tearDown(client: S3Client) { + client.deleteObject { + bucket = bucketName + key = KEY + } + } + } +} diff --git a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt index e05c7a29e54..b999b4839a1 100644 --- a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt +++ b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt @@ -14,12 +14,12 @@ import aws.smithy.kotlin.runtime.telemetry.metrics.* import aws.smithy.kotlin.runtime.telemetry.trace.TracerProvider private val capturedMetrics = mapOf( - "smithy.client.attempt_overhead_duration" to "Overhead", +// "smithy.client.call.attempt_overhead_duration" to "Overhead", // "smithy.client.http.time_to_first_byte" to "TTFB", - // "smithy.client.attempt_duration" to "Call", - // "smithy.client.serialization_duration" to "Serlz", - // "smithy.client.deserialization_duration" to "Deserlz", - // "smithy.client.resolve_endpoint_duration" to "EPR", + "smithy.client.call.attempt_duration" to "Call", + // "smithy.client.call.serialization_duration" to "Serlz", + // "smithy.client.call.deserialization_duration" to "Deserlz", + // "smithy.client.call.resolve_endpoint_duration" to "EPR", ) @ExperimentalApi From b1ea753ccfaa63ff56072b735414c179d445b4b9 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 9 Feb 2024 14:05:17 -0500 Subject: [PATCH 035/145] Re-enable benchmarks --- .../benchmarks/service/BenchmarkHarness.kt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt index 25189591afe..ae60c0a2437 100644 --- a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt +++ b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt @@ -15,14 +15,14 @@ val DEFAULT_WARMUP_TIME = 5.seconds val DEFAULT_ITERATION_TIME = 15.seconds private val benchmarks = setOf( -// S3Benchmark(), -// SnsBenchmark(), -// StsBenchmark(), -// CloudwatchBenchmark(), -// CloudwatchEventsBenchmark(), -// DynamoDbBenchmark(), -// PinpointBenchmark(), - S3ExpressBenchmark() + S3Benchmark(), + SnsBenchmark(), + StsBenchmark(), + CloudwatchBenchmark(), + CloudwatchEventsBenchmark(), + DynamoDbBenchmark(), + PinpointBenchmark(), + S3ExpressBenchmark(), ).map { @Suppress("UNCHECKED_CAST") it as ServiceBenchmark From d504c5524b739df978f891eabc657c362844c572 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 9 Feb 2024 14:06:28 -0500 Subject: [PATCH 036/145] revert capturedMetrics --- .../service/telemetry/BenchmarkTelemetryProvider.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt index b999b4839a1..5a031679b83 100644 --- a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt +++ b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt @@ -14,9 +14,9 @@ import aws.smithy.kotlin.runtime.telemetry.metrics.* import aws.smithy.kotlin.runtime.telemetry.trace.TracerProvider private val capturedMetrics = mapOf( -// "smithy.client.call.attempt_overhead_duration" to "Overhead", + "smithy.client.call.attempt_overhead_duration" to "Overhead", // "smithy.client.http.time_to_first_byte" to "TTFB", - "smithy.client.call.attempt_duration" to "Call", + // "smithy.client.call.attempt_duration" to "Call", // "smithy.client.call.serialization_duration" to "Serlz", // "smithy.client.call.deserialization_duration" to "Deserlz", // "smithy.client.call.resolve_endpoint_duration" to "EPR", From 8aedf25bbb827b46567d34d17e24132905fe6f36 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 9 Feb 2024 14:06:40 -0500 Subject: [PATCH 037/145] ktlintFormat --- .../kotlin/services/s3/S3ExpressCredentialsProvider.kt | 2 -- .../s3/internal/SdkS3ExpressCredentialsProvider.kt | 2 +- .../service/definitions/S3ExpressBenchmark.kt | 10 ++++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt index 1f843d7596b..1051279fc05 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt @@ -12,5 +12,3 @@ public interface S3ExpressCredentialsProvider : CloseableCredentialsProvider { override suspend fun resolve(attributes: Attributes): Credentials override fun close() } - - diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt index 959e314b7a5..f578154de4b 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt @@ -33,4 +33,4 @@ public class SdkS3ExpressCredentialsProvider( override fun close() { credentialsCache.close() } -} \ No newline at end of file +} diff --git a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt index 2ae21d7ce65..6fac3f2fff4 100644 --- a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt +++ b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt @@ -81,10 +81,12 @@ class S3ExpressBenchmark : ServiceBenchmark { } override suspend fun transact(client: S3Client) { - client.getObject(GetObjectRequest { - bucket = bucketName - key = KEY - }) { } + client.getObject( + GetObjectRequest { + bucket = bucketName + key = KEY + }, + ) { } } override suspend fun tearDown(client: S3Client) { From 02aaa9255cb88358e8994d4fd1edae4abfea641c Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 9 Feb 2024 14:10:15 -0500 Subject: [PATCH 038/145] remove println --- .../s3/internal/S3ExpressCredentialsCache.kt | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index c217397417f..bfe2c1e33f9 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -58,20 +58,15 @@ public class S3ExpressCredentialsCache( private suspend fun refresh() { val logger = coroutineContext.logger() while (isActive) { - logger.trace { "Looping..." } - println("Looping...") val refreshedCredentials = mutableMapOf>() var nextRefresh: Instant = clock.now() + DEFAULT_REFRESH_PERIOD lru.withLock { lru.entries.forEach { (key, cachedValue) -> - logger.trace { "Checking entry for ${key.bucket}" } - println("Checking entry for ${key.bucket}") nextRefresh = minOf(nextRefresh, cachedValue.expiresAt - REFRESH_BUFFER) if (cachedValue.isExpired(clock)) { - logger.trace { "Credentials for ${key.bucket} expire within the refresh buffer period, performing a refresh..." } - println("Credentials for ${key.bucket} expire within the refresh buffer period, performing a refresh...") + logger.debug { "Credentials for ${key.bucket} expire within the refresh buffer period, performing a refresh..." } createSessionCredentials(key).also { refreshedCredentials[key] = it nextRefresh = minOf(nextRefresh, it.expiresAt - REFRESH_BUFFER) @@ -87,14 +82,8 @@ public class S3ExpressCredentialsCache( // wake up when it's time to refresh or an immediate refresh has been triggered select { - onTimeout(clock.now().until(nextRefresh)) { - logger.trace { "Woke up from timeout" } - println("Woke up from timeout") - } - immediateRefreshChannel.onReceive { - logger.trace { "Woke up from channel" } - println("Woke up from channel") - } + onTimeout(clock.now().until(nextRefresh)) {} + immediateRefreshChannel.onReceive {} } } } @@ -120,7 +109,7 @@ public class S3ExpressCredentialsCache( providerName = "S3ExpressCredentialsProvider", ), credentials.expiration, - ).also { logger.debug { "got credentials ${it.value}" } } + ) } override fun close() { From a7f3b577a9a8fa10441ce759964d843a67e3d15c Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 9 Feb 2024 14:14:24 -0500 Subject: [PATCH 039/145] update endpoint attribute check --- .../http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt | 5 +++-- .../http/interceptors/S3ExpressDisableChecksumInterceptor.kt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt index 5b323ff0596..e3c4b625712 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt @@ -13,14 +13,15 @@ import aws.smithy.kotlin.runtime.http.request.toBuilder import aws.smithy.kotlin.runtime.telemetry.logging.logger import kotlin.coroutines.coroutineContext -internal val S3_EXPRESS_ENDPOINT_PROPERTY = "backend" +internal val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" +internal val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" private val CRC32_ALGORITHM_NAME = "CRC32" public class S3ExpressCrc32ChecksumInterceptor( public val checksumAlgorithmHeaderName: String? = null, ) : HttpInterceptor { override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { - if (!context.executionContext.contains(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY))) { + if (context.executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE) { return context.protocolRequest } diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt index e5d6b7f4f25..191d44a3b17 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt @@ -17,7 +17,7 @@ import kotlin.coroutines.coroutineContext */ public class S3ExpressDisableChecksumInterceptor : HttpInterceptor { override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { - if (!context.executionContext.contains(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY))) { + if (context.executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE) { return context.protocolRequest } From 25b1387e87f02b5db166a365a231628263d2b0fe Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 9 Feb 2024 14:15:40 -0500 Subject: [PATCH 040/145] Kdocs --- .../aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt index 1051279fc05..492e4588ed6 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt @@ -8,6 +8,9 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvide import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.collections.Attributes +/** + * A credentials provider used for making requests to S3 Express One Zone directory buckets. + */ public interface S3ExpressCredentialsProvider : CloseableCredentialsProvider { override suspend fun resolve(attributes: Attributes): Credentials override fun close() From 35bfe704b98318cad46000297cfd8547a37a3a24 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 9 Feb 2024 16:08:03 -0500 Subject: [PATCH 041/145] CI From 5475c107ea498615800dcf7b814bbc1c4fde5a1f Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Mon, 12 Feb 2024 17:35:03 -0500 Subject: [PATCH 042/145] Fix header name --- .../src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt index 8620a819daa..2f82d315ce4 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt @@ -12,7 +12,7 @@ import aws.smithy.kotlin.runtime.http.auth.HttpSigner import aws.smithy.kotlin.runtime.http.auth.SignHttpRequest import aws.smithy.kotlin.runtime.http.request.header -private const val S3_EXPRESS_SESSION_TOKEN_HEADER = "X-Amz-S3Session-Token" +private const val S3_EXPRESS_SESSION_TOKEN_HEADER = "X-Amz-S3session-Token" private const val SESSION_TOKEN_HEADER = "X-Amz-Security-Token" /** From 18d594ff4cfafedbea4c08d2efe725deb54d52ab Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 11:04:15 -0500 Subject: [PATCH 043/145] Remove explicit type --- .../services/s3/internal/SdkS3ExpressCredentialsProvider.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt index f578154de4b..903a528dba2 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt @@ -22,7 +22,7 @@ public class SdkS3ExpressCredentialsProvider( override suspend fun resolve(attributes: Attributes): Credentials { val logger = coroutineContext.logger() - val bucket: String = attributes[S3ExpressAttributes.Bucket] + val bucket = attributes[S3ExpressAttributes.Bucket] val client = attributes[S3ExpressAttributes.Client] val key = S3ExpressCredentialsCacheKey(bucket, client, bootstrapCredentialsProvider.resolve(attributes)) From 8d86e72ed4fdd2c42e0feff1a1488bc6e31656c2 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 12:45:40 -0500 Subject: [PATCH 044/145] Simplify `S3ExpressCrc32ChecksumInterceptor` installation --- .../S3ExpressCrc32ChecksumInterceptor.kt | 25 +++++++++++-------- .../s3/express/S3ExpressIntegration.kt | 8 +----- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt index e3c4b625712..e631d51823d 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt @@ -28,17 +28,20 @@ public class S3ExpressCrc32ChecksumInterceptor( val logger = coroutineContext.logger() val req = context.protocolRequest.toBuilder() - // Update the execution context so flexible checksums uses CRC32 - logger.info { "Setting checksum algorithm to $CRC32_ALGORITHM_NAME for S3 Express" } - context.executionContext[HttpOperationContext.ChecksumAlgorithm] = CRC32_ALGORITHM_NAME - - // Most checksum headers are handled by the flexible checksums feature. But, S3 models an HTTP header binding for the - // checksum algorithm, which also needs to be overwritten and set to CRC32. - // - // The header is already set by the time this interceptor runs, so it needs to be overwritten and can't be set - // through the normal path. - checksumAlgorithmHeaderName?.let { - req.headers[it] = CRC32_ALGORITHM_NAME + if (!context.executionContext.contains(HttpOperationContext.ChecksumAlgorithm)) { + logger.info { "Checksum is required and not already configured, enabling CRC32 for S3 Express" } + + // Update the execution context so flexible checksums uses CRC32 + context.executionContext[HttpOperationContext.ChecksumAlgorithm] = CRC32_ALGORITHM_NAME + + // Most checksum headers are handled by the flexible checksums feature. But, S3 models an HTTP header binding for the + // checksum algorithm, which also needs to be overwritten and set to CRC32. + // + // The header is already set by the time this interceptor runs, so it needs to be overwritten and can't be set + // through the normal path. + checksumAlgorithmHeaderName?.let { + req.headers[it] = CRC32_ALGORITHM_NAME + } } return req.build() diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 960a4fad525..87c07bc37a2 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -118,13 +118,7 @@ class S3ExpressIntegration : KotlinIntegration { val checksumRequired = op.hasTrait() || httpChecksumTrait?.isRequestChecksumRequired == true if (checksumRequired) { - if (checksumAlgorithmMember != null) { // enable flexible checksums using CRC32 if the user has not already opted-in - writer.withBlock("if (input.${checksumAlgorithmMember.defaultName()} == null) {", "}") { - writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq()}))", AwsRuntimeTypes.Http.Interceptors.S3ExpressCrc32ChecksumInterceptor) - } - } else { // checksum required, operation does not expose flexible checksums member, so set the checksum algorithm manually - writer.write("op.interceptors.add(#T())", AwsRuntimeTypes.Http.Interceptors.S3ExpressCrc32ChecksumInterceptor) - } + writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq() ?: ""}))", AwsRuntimeTypes.Http.Interceptors.S3ExpressCrc32ChecksumInterceptor) } } } From c5da75492bb0cc90ef10de0142f30afe145819ad Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 12:54:17 -0500 Subject: [PATCH 045/145] Remove redundant `.invoke` --- .../kotlin/services/s3/internal/S3ExpressCredentialsCache.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index bfe2c1e33f9..a8a3d56bf5b 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -101,7 +101,7 @@ public class S3ExpressCredentialsCache( } return ExpiringValue( - Credentials.invoke( + Credentials( accessKeyId = credentials.accessKeyId, secretAccessKey = credentials.secretAccessKey, sessionToken = credentials.sessionToken, From 39dbeb61eb131f47493d240c39c040578f94e43d Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 13:34:27 -0500 Subject: [PATCH 046/145] Don't deconfigure credentials --- .../services/s3/internal/S3ExpressCredentialsCache.kt | 11 +---------- services/s3/e2eTest/src/S3ExpressTest.kt | 7 +++++-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index a8a3d56bf5b..fbee8531bef 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -89,16 +89,7 @@ public class S3ExpressCredentialsCache( } private suspend fun createSessionCredentials(key: S3ExpressCredentialsCacheKey): ExpiringValue { - val logger = coroutineContext.logger() - - // FIXME Consider creating a brand new client using the bootstrapped key.credentials instead of abusing the user's client - val credentials = (key.client as S3Client) - // de-configure interceptors because this key.client is the user's S3 client, and we don't want to - // execute their custom interceptors during this internal createSession request - .withConfig { interceptors = mutableListOf() } - .use { - it.createSession { bucket = key.bucket }.credentials!! - } + val credentials = (key.client as S3Client).createSession { bucket = key.bucket }.credentials!! return ExpiringValue( Credentials( diff --git a/services/s3/e2eTest/src/S3ExpressTest.kt b/services/s3/e2eTest/src/S3ExpressTest.kt index bd5097cc6a4..e05c9e65659 100644 --- a/services/s3/e2eTest/src/S3ExpressTest.kt +++ b/services/s3/e2eTest/src/S3ExpressTest.kt @@ -9,6 +9,7 @@ import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.putObject import aws.sdk.kotlin.services.s3.withConfig import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext +import aws.smithy.kotlin.runtime.collections.AttributeKey import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.decodeToString import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor @@ -108,8 +109,10 @@ class S3ExpressTest { private class CRC32ChecksumValidatingInterceptor : HttpInterceptor { override fun readAfterSigning(context: ProtocolRequestInterceptorContext) { val headers = context.protocolRequest.headers - assertTrue(headers.contains("x-amz-checksum-crc32"), "Failed to find x-amz-checksum-crc32 header") - assertFalse(headers.contains("Content-MD5"), "Unexpectedly found Content-MD5 header") + if (headers.contains("X-Amz-S3session-Token")) { + assertTrue(headers.contains("x-amz-checksum-crc32"), "Failed to find x-amz-checksum-crc32 header") + assertFalse(headers.contains("Content-MD5"), "Unexpectedly found Content-MD5 header") + } } } } From 700eade5f3d2a032a2e049b98af87de7e702f0c3 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 14:11:50 -0500 Subject: [PATCH 047/145] Refactor S3ExpressAttributes --- aws-runtime/aws-config/api/aws-config.api | 6 ------ .../runtime/auth/S3ExpressAttributes.kt | 13 ------------ .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 2 +- .../s3/express/S3ExpressIntegration.kt | 15 +++++++++---- .../sdk/kotlin/services/s3/S3Attributes.kt | 21 +++++++++++++++++++ .../SdkS3ExpressCredentialsProvider.kt | 6 +++--- 6 files changed, 36 insertions(+), 27 deletions(-) delete mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt create mode 100644 services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index 6a110e51c31..fce28564610 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -1,9 +1,3 @@ -public final class aws/sdk/kotlin/runtime/auth/S3ExpressAttributes { - public static final field INSTANCE Laws/sdk/kotlin/runtime/auth/S3ExpressAttributes; - public final fun getBucket ()Laws/smithy/kotlin/runtime/collections/AttributeKey; - public final fun getClient ()Laws/smithy/kotlin/runtime/collections/AttributeKey; -} - public final class aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme : aws/smithy/kotlin/runtime/http/auth/AuthScheme { public fun (Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner;Ljava/lang/String;)V public synthetic fun (Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt deleted file mode 100644 index fc7756211de..00000000000 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/S3ExpressAttributes.kt +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.runtime.auth - -import aws.smithy.kotlin.runtime.client.SdkClient -import aws.smithy.kotlin.runtime.collections.AttributeKey - -public object S3ExpressAttributes { - public val Bucket: AttributeKey = AttributeKey("aws.sdk.kotlin#S3ExpressBucket") - public val Client: AttributeKey = AttributeKey("aws.sdk.kotlin#S3ExpressClient") -} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index dfb40317256..d3ed1c286a4 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -58,7 +58,7 @@ object AwsRuntimeTypes { object Auth : RuntimeTypePackage(AwsKotlinDependency.AWS_CONFIG, "auth") { val SigV4S3ExpressAuthScheme = symbol("SigV4S3ExpressAuthScheme") val sigV4S3Express = symbol("sigV4S3Express") - val S3ExpressAttributes = symbol("S3ExpressAttributes") + } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 87c07bc37a2..9bd2c8a754b 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -13,6 +13,7 @@ import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding +import software.amazon.smithy.kotlin.codegen.model.buildSymbol import software.amazon.smithy.kotlin.codegen.model.expectShape import software.amazon.smithy.kotlin.codegen.model.getTrait import software.amazon.smithy.kotlin.codegen.model.hasTrait @@ -75,8 +76,11 @@ class S3ExpressIntegration : KotlinIntegration { override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = ctx.model.expectShape(ctx.settings.service).isS3 override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val attributesSymbol = AwsRuntimeTypes.Config.Auth.S3ExpressAttributes - writer.write("op.context[#T.Client] = this", attributesSymbol) + val attributesSymbol = buildSymbol { + name = "S3Attributes" + namespace = "aws.sdk.kotlin.services.s3" + } + writer.write("op.context[#T.ExpressClient] = this", attributesSymbol) } } @@ -89,8 +93,11 @@ class S3ExpressIntegration : KotlinIntegration { .any { it.memberName == "Bucket" } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val attributesSymbol = AwsRuntimeTypes.Config.Auth.S3ExpressAttributes - writer.write("input.bucket?.let { op.context[#T.Bucket] = it }", attributesSymbol) + val attributesSymbol = buildSymbol { + name = "S3Attributes" + namespace = "aws.sdk.kotlin.services.s3" + } + writer.write("input.bucket?.let { op.context[#T.DirectoryBucket] = it }", attributesSymbol) } } diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt new file mode 100644 index 00000000000..fc45123ea1d --- /dev/null +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt @@ -0,0 +1,21 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.services.s3 + +import aws.smithy.kotlin.runtime.client.SdkClient +import aws.smithy.kotlin.runtime.collections.AttributeKey + +public object S3Attributes { + /** + * The name of the directory bucket requests are being made to + */ + public val DirectoryBucket: AttributeKey = AttributeKey("aws.sdk.kotlin#S3ExpressBucket") + + /** + * The S3 client being used to make requests to directory buckets. This client will be used to call s3:CreateSession + * to obtain directory bucket credentials. + */ + public val ExpressClient: AttributeKey = AttributeKey("aws.sdk.kotlin#S3ExpressClient") +} diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt index 903a528dba2..a77cbc17fef 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt @@ -4,7 +4,7 @@ */ package aws.sdk.kotlin.services.s3.internal -import aws.sdk.kotlin.runtime.auth.S3ExpressAttributes +import aws.sdk.kotlin.services.s3.S3Attributes import aws.sdk.kotlin.services.s3.S3ExpressCredentialsProvider import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider @@ -22,8 +22,8 @@ public class SdkS3ExpressCredentialsProvider( override suspend fun resolve(attributes: Attributes): Credentials { val logger = coroutineContext.logger() - val bucket = attributes[S3ExpressAttributes.Bucket] - val client = attributes[S3ExpressAttributes.Client] + val bucket = attributes[S3Attributes.DirectoryBucket] + val client = attributes[S3Attributes.ExpressClient] val key = S3ExpressCredentialsCacheKey(bucket, client, bootstrapCredentialsProvider.resolve(attributes)) From 204d13ef1db2123712d214f98bf4c4ef1f57e6cf Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 15:18:12 -0500 Subject: [PATCH 048/145] Relocate SigV4S3ExpressAuthScheme --- aws-runtime/aws-config/api/aws-config.api | 14 -------------- .../services/s3}/SigV4S3ExpressAuthScheme.kt | 9 +++++---- 2 files changed, 5 insertions(+), 18 deletions(-) rename {aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth => services/s3/common/src/aws/sdk/kotlin/services/s3}/SigV4S3ExpressAuthScheme.kt (92%) diff --git a/aws-runtime/aws-config/api/aws-config.api b/aws-runtime/aws-config/api/aws-config.api index fce28564610..62be3ef2207 100644 --- a/aws-runtime/aws-config/api/aws-config.api +++ b/aws-runtime/aws-config/api/aws-config.api @@ -1,17 +1,3 @@ -public final class aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme : aws/smithy/kotlin/runtime/http/auth/AuthScheme { - public fun (Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner;Ljava/lang/String;)V - public synthetic fun (Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V - public fun (Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner;)V - public fun getSchemeId-DepwgT4 ()Ljava/lang/String; - public fun getSigner ()Laws/smithy/kotlin/runtime/http/auth/HttpSigner; - public fun identityProvider (Laws/smithy/kotlin/runtime/identity/IdentityProviderConfig;)Laws/smithy/kotlin/runtime/identity/IdentityProvider; -} - -public final class aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthSchemeKt { - public static final fun sigV4S3Express (ZLjava/lang/String;Ljava/lang/String;Ljava/lang/Boolean;Ljava/lang/Boolean;)Laws/smithy/kotlin/runtime/auth/AuthOption; - public static synthetic fun sigV4S3Express$default (ZLjava/lang/String;Ljava/lang/String;Ljava/lang/Boolean;Ljava/lang/Boolean;ILjava/lang/Object;)Laws/smithy/kotlin/runtime/auth/AuthOption; -} - public final class aws/sdk/kotlin/runtime/auth/credentials/AssumeRoleParameters { public synthetic fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;JLjava/util/List;Ljava/lang/String;Ljava/util/Map;Ljava/util/List;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public synthetic fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;JLjava/util/List;Ljava/lang/String;Ljava/util/Map;Ljava/util/List;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/internal/DefaultConstructorMarker;)V diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt similarity index 92% rename from aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt index 70108e9f0e8..e86d2ce8785 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/auth/SigV4S3ExpressAuthScheme.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.runtime.auth +package aws.sdk.kotlin.services.s3 import aws.sdk.kotlin.runtime.http.S3ExpressHttpSigner import aws.smithy.kotlin.runtime.InternalApi @@ -21,7 +21,8 @@ import aws.smithy.kotlin.runtime.http.auth.AuthScheme import aws.smithy.kotlin.runtime.http.auth.AwsHttpSigner import aws.smithy.kotlin.runtime.http.auth.HttpSigner -private val S3_EXPRESS_AUTH_SCHEME_ID = AuthSchemeId("aws.auth#sigv4s3express") +public val AuthSchemeId.Companion.AwsSigV4S3Express: AuthSchemeId + get() = AuthSchemeId("aws.auth#sigv4s3express") /** * HTTP auth scheme for S3 Express One Zone authentication @@ -39,7 +40,7 @@ public class SigV4S3ExpressAuthScheme( ), ) - override val schemeId: AuthSchemeId = S3_EXPRESS_AUTH_SCHEME_ID + override val schemeId: AuthSchemeId = AuthSchemeId.AwsSigV4S3Express override val signer: HttpSigner = S3ExpressHttpSigner(awsHttpSigner) } @@ -68,7 +69,7 @@ public fun sigV4S3Express( } else { emptyAttributes() } - return AuthOption(S3_EXPRESS_AUTH_SCHEME_ID, attrs) + return AuthOption(AuthSchemeId.AwsSigV4S3Express, attrs) } // FIXME Copied from SigV4AuthScheme.kt From 6605462ab29019a8f5118d64cdd7b9f4bf3fb389 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 16:35:59 -0500 Subject: [PATCH 049/145] Fix symbols usage --- .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 6 ------ .../s3/express/S3ExpressIntegration.kt | 4 ++-- .../SigV4S3ExpressAuthSchemeIntegration.kt | 18 ++++++++++++++---- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index d3ed1c286a4..b51acb0cdb1 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -54,12 +54,6 @@ object AwsRuntimeTypes { val StaticCredentialsProvider = symbol("StaticCredentialsProvider") val manage = symbol("manage", "auth.credentials.internal", isExtension = true) } - - object Auth : RuntimeTypePackage(AwsKotlinDependency.AWS_CONFIG, "auth") { - val SigV4S3ExpressAuthScheme = symbol("SigV4S3ExpressAuthScheme") - val sigV4S3Express = symbol("sigV4S3Express") - - } } object Http : RuntimeTypePackage(AwsKotlinDependency.AWS_HTTP) { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 9bd2c8a754b..fd970290825 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -50,12 +50,12 @@ class S3ExpressIntegration : KotlinIntegration { private val configureS3ExpressAuthSchemeWriter = AppendingSectionWriter { writer -> writer.withBlock("getOrPut(#T) {", "}", SigV4S3ExpressAuthSchemeHandler().authSchemeIdSymbol) { - writer.write("#T(#T, #S)", AwsRuntimeTypes.Config.Auth.SigV4S3ExpressAuthScheme, RuntimeTypes.Auth.Signing.AwsSigningStandard.DefaultAwsSigner, "s3") + writer.write("#T(#T, #S)", SigV4S3ExpressAuthSchemeSymbol, RuntimeTypes.Auth.Signing.AwsSigningStandard.DefaultAwsSigner, "s3") } } private val setServiceDefaultAuthOptionWriter = AppendingSectionWriter { writer -> - writer.write("#T(),", AwsRuntimeTypes.Config.Auth.sigV4S3Express) + writer.write("#T(),", sigV4S3ExpressSymbol) } private val configureIdentityProviderForAuthSchemeWriter = AppendingSectionWriter { writer -> diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index d91ba06e626..93c7697def3 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -52,6 +52,16 @@ class SigV4S3ExpressAuthSchemeIntegration : SigV4AuthSchemeIntegration() { override fun authSchemes(ctx: ProtocolGenerator.GenerationContext): List = listOf(SigV4S3ExpressAuthSchemeHandler()) } +internal val sigV4S3ExpressSymbol = buildSymbol { + name = "sigV4S3Express" + namespace = "aws.sdk.kotlin.services.s3" +} + +internal val SigV4S3ExpressAuthSchemeSymbol = buildSymbol { + name = "SigV4S3ExpressAuthScheme" + namespace = "aws.sdk.kotlin.services.s3" +} + private object SigV4S3ExpressEndpointCustomization : EndpointCustomization { override val propertyRenderers: Map = mapOf( "authSchemes" to ::renderAuthSchemes, @@ -62,7 +72,7 @@ open class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { override val authSchemeId: ShapeId = ShapeId.from("aws.auth#sigv4s3express") override val authSchemeIdSymbol: Symbol = buildSymbol { - name = "AuthSchemeId(\"aws.auth#sigv4s3express\")" + name = "AuthSchemeId.AwsSigV4S3Express" val ref = RuntimeTypes.Auth.Identity.AuthSchemeId objectRef = ref namespace = ref.namespace @@ -83,12 +93,12 @@ open class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { } else { "#T()" } - writer.write(expr, AwsRuntimeTypes.Config.Auth.sigV4S3Express) + writer.write(expr, sigV4S3ExpressSymbol) } override fun instantiateAuthSchemeExpr(ctx: ProtocolGenerator.GenerationContext, writer: KotlinWriter) { val signingService = AwsSignatureVersion4.signingServiceName(ctx.service) - writer.write("#T(#T, #S)", AwsRuntimeTypes.Config.Auth.SigV4S3ExpressAuthScheme, RuntimeTypes.Auth.Signing.AwsSigningStandard.DefaultAwsSigner, signingService) + writer.write("#T(#T, #S)", SigV4S3ExpressAuthSchemeSymbol, RuntimeTypes.Auth.Signing.AwsSigningStandard.DefaultAwsSigner, signingService) } } @@ -98,7 +108,7 @@ private fun renderAuthSchemes(writer: KotlinWriter, authSchemes: Expression, exp authSchemes.toNode().expectArrayNode().forEach { val scheme = it.expectObjectNode() val schemeName = scheme.expectStringMember("name").value - val authFactoryFn = if (schemeName == "sigv4-s3express") AwsRuntimeTypes.Config.Auth.sigV4S3Express else return@forEach + val authFactoryFn = if (schemeName == "sigv4-s3express") sigV4S3ExpressSymbol else return@forEach withBlock("#T(", "),", authFactoryFn) { // we delegate back to the expression visitor for each of these fields because it's possible to From d4f07efa35c1f4998e738e56712574b6101c3ea8 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 16:43:05 -0500 Subject: [PATCH 050/145] Simplify attributes --- .../services/s3/SigV4S3ExpressAuthScheme.kt | 44 ++----------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt index e86d2ce8785..bbe12adabae 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt @@ -11,15 +11,10 @@ import aws.smithy.kotlin.runtime.auth.AuthOption import aws.smithy.kotlin.runtime.auth.AuthSchemeId import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigner import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAlgorithm -import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAttributes -import aws.smithy.kotlin.runtime.auth.awssigning.HashSpecification -import aws.smithy.kotlin.runtime.collections.AttributeKey -import aws.smithy.kotlin.runtime.collections.MutableAttributes -import aws.smithy.kotlin.runtime.collections.emptyAttributes -import aws.smithy.kotlin.runtime.collections.mutableAttributes import aws.smithy.kotlin.runtime.http.auth.AuthScheme import aws.smithy.kotlin.runtime.http.auth.AwsHttpSigner import aws.smithy.kotlin.runtime.http.auth.HttpSigner +import aws.smithy.kotlin.runtime.http.auth.sigV4 public val AuthSchemeId.Companion.AwsSigV4S3Express: AuthSchemeId get() = AuthSchemeId("aws.auth#sigv4s3express") @@ -61,38 +56,7 @@ public fun sigV4S3Express( disableDoubleUriEncode: Boolean? = null, normalizeUriPath: Boolean? = null, ): AuthOption { - val attrs = if (unsignedPayload || serviceName != null || signingRegion != null || disableDoubleUriEncode != null || normalizeUriPath != null) { - val mutAttrs = mutableAttributes() - mutAttrs.setNotBlank(AwsSigningAttributes.SigningRegion, signingRegion) - setCommonSigV4Attrs(mutAttrs, unsignedPayload, serviceName, disableDoubleUriEncode, normalizeUriPath) - mutAttrs - } else { - emptyAttributes() - } - return AuthOption(AuthSchemeId.AwsSigV4S3Express, attrs) -} - -// FIXME Copied from SigV4AuthScheme.kt -internal fun MutableAttributes.setNotBlank(key: AttributeKey, value: String?) { - if (!value.isNullOrBlank()) set(key, value) -} - -internal fun setCommonSigV4Attrs( - attrs: MutableAttributes, - unsignedPayload: Boolean = false, - serviceName: String? = null, - disableDoubleUriEncode: Boolean? = null, - normalizeUriPath: Boolean? = null, -) { - if (unsignedPayload) { - attrs[AwsSigningAttributes.HashSpecification] = HashSpecification.UnsignedPayload - } - attrs.setNotBlank(AwsSigningAttributes.SigningService, serviceName) - if (disableDoubleUriEncode != null) { - attrs[AwsSigningAttributes.UseDoubleUriEncode] = !disableDoubleUriEncode - } - - if (normalizeUriPath != null) { - attrs[AwsSigningAttributes.NormalizeUriPath] = normalizeUriPath - } + // Note: SigV4-S3Express has the same attributes as SigV4 + val sigV4AuthOption = sigV4(unsignedPayload, serviceName, signingRegion, disableDoubleUriEncode, normalizeUriPath) + return AuthOption(AuthSchemeId.AwsSigV4S3Express, sigV4AuthOption.attributes) } From 22e5433eb6a69f3bb8aa873a3bd6c0ba34a4760d Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 17:33:41 -0500 Subject: [PATCH 051/145] const vals --- .../http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt index e631d51823d..563345bd11f 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt @@ -13,9 +13,9 @@ import aws.smithy.kotlin.runtime.http.request.toBuilder import aws.smithy.kotlin.runtime.telemetry.logging.logger import kotlin.coroutines.coroutineContext -internal val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" -internal val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" -private val CRC32_ALGORITHM_NAME = "CRC32" +internal const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" +internal const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" +private const val CRC32_ALGORITHM_NAME = "CRC32" public class S3ExpressCrc32ChecksumInterceptor( public val checksumAlgorithmHeaderName: String? = null, From 629f9377f26083809e0c0c7848d354e33b4dd23c Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 17:34:34 -0500 Subject: [PATCH 052/145] Bump log level and add operation name --- .../http/interceptors/S3ExpressDisableChecksumInterceptor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt index 191d44a3b17..aedc5c1bed0 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt @@ -26,7 +26,7 @@ public class S3ExpressDisableChecksumInterceptor : HttpInterceptor { val configuredChecksumAlgorithm = context.executionContext.getOrNull(HttpOperationContext.ChecksumAlgorithm) configuredChecksumAlgorithm?.let { - logger.info { "Disabling configured checksum $it for S3 Express" } + logger.warn { "Disabling configured checksum $it for S3 Express UploadPart" } context.executionContext.remove(HttpOperationContext.ChecksumAlgorithm) } From eea0bab878b26c80156c0a72d10ec39b0ec41d56 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 17:47:08 -0500 Subject: [PATCH 053/145] Make cache and key internal, key data class --- .../s3/internal/S3ExpressCredentialsCache.kt | 33 +++++-------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index fbee8531bef..fbb66f56062 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -9,7 +9,6 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.collections.LruCache import aws.smithy.kotlin.runtime.io.Closeable -import aws.smithy.kotlin.runtime.io.use import aws.smithy.kotlin.runtime.telemetry.logging.logger import aws.smithy.kotlin.runtime.time.Clock import aws.smithy.kotlin.runtime.time.Instant @@ -25,7 +24,7 @@ private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 private val REFRESH_BUFFER = 1.minutes private val DEFAULT_REFRESH_PERIOD = 5.minutes -public class S3ExpressCredentialsCache( +internal class S3ExpressCredentialsCache( private val clock: Clock = Clock.System, ) : CoroutineScope, Closeable { override val coroutineContext: CoroutineContext = Job() + CoroutineName("S3ExpressCredentialsCacheRefresh") @@ -39,10 +38,10 @@ public class S3ExpressCredentialsCache( } } - public suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru.get(key)?.takeIf { !it.isExpired(clock) }?.value + suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru.get(key)?.takeIf { !it.isExpired(clock) }?.value ?: (createSessionCredentials(key).also { put(key, it) }).value - public suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue) { + suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue) { lru.put(key, value) immediateRefreshChannel.send(Unit) } @@ -109,24 +108,8 @@ public class S3ExpressCredentialsCache( } } -public class S3ExpressCredentialsCacheKey( - public val bucket: String, - public val client: SdkClient, - public val credentials: Credentials, -) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other !is S3ExpressCredentialsCacheKey) return false - if (bucket != other.bucket) return false - if (client != other.client) return false - if (credentials != other.credentials) return false - return true - } - - override fun hashCode(): Int { - var result = bucket.hashCode() ?: 0 - result = 31 * result + (client.hashCode() ?: 0) - result = 31 * result + (credentials.hashCode() ?: 0) - return result - } -} +internal data class S3ExpressCredentialsCacheKey( + val bucket: String, + val client: SdkClient, + val credentials: Credentials, +) From 4d64f9026d7cc8385f16e11b0f61c61d6e3e3990 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 13 Feb 2024 18:07:17 -0500 Subject: [PATCH 054/145] Rename to DefaultS3ExpressCredentialsProvider, make internal and expose through interface companion object --- .../codegen/customization/s3/ClientConfigIntegration.kt | 6 +++--- .../kotlin/services/s3/S3ExpressCredentialsProvider.kt | 7 +++++++ .../s3/internal/SdkS3ExpressCredentialsProvider.kt | 8 ++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index 5729b64e2ca..c52f17f9448 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -74,9 +74,9 @@ class ClientConfigIntegration : KotlinIntegration { val S3ExpressCredentialsProvider: ConfigProperty = ConfigProperty { name = "s3ExpressCredentialsProvider" symbol = buildSymbol { - name = "SdkS3ExpressCredentialsProvider" + name = "S3ExpressCredentialsProvider" nullable = false - namespace = "aws.sdk.kotlin.services.s3.internal" + namespace = "aws.sdk.kotlin.services.s3" } documentation = """ Credentials provider to be used for making requests to S3 Express. @@ -92,7 +92,7 @@ class ClientConfigIntegration : KotlinIntegration { propertyType = ConfigPropertyType.Custom( render = { _, writer -> - writer.write("public val $name: S3ExpressCredentialsProvider = builder.$name ?: SdkS3ExpressCredentialsProvider(this.credentialsProvider)") + writer.write("public val $name: S3ExpressCredentialsProvider = builder.$name ?: S3ExpressCredentialsProvider.default(this.credentialsProvider)") }, renderBuilder = { prop, writer -> prop.documentation?.let(writer::dokka) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt index 492e4588ed6..80ea856fb0b 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt @@ -4,8 +4,10 @@ */ package aws.sdk.kotlin.services.s3 +import aws.sdk.kotlin.services.s3.internal.DefaultS3ExpressCredentialsProvider import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvider import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider import aws.smithy.kotlin.runtime.collections.Attributes /** @@ -14,4 +16,9 @@ import aws.smithy.kotlin.runtime.collections.Attributes public interface S3ExpressCredentialsProvider : CloseableCredentialsProvider { override suspend fun resolve(attributes: Attributes): Credentials override fun close() + + public companion object { + public fun default(bootstrapCredentialsProvider: CredentialsProvider): S3ExpressCredentialsProvider = + DefaultS3ExpressCredentialsProvider(bootstrapCredentialsProvider) + } } diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt index a77cbc17fef..495faed85ea 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt @@ -11,17 +11,13 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.io.SdkManagedBase -import aws.smithy.kotlin.runtime.telemetry.logging.logger -import kotlin.coroutines.coroutineContext -public class SdkS3ExpressCredentialsProvider( - public val bootstrapCredentialsProvider: CredentialsProvider, +internal class DefaultS3ExpressCredentialsProvider( + val bootstrapCredentialsProvider: CredentialsProvider, ) : S3ExpressCredentialsProvider, SdkManagedBase() { private val credentialsCache = S3ExpressCredentialsCache() override suspend fun resolve(attributes: Attributes): Credentials { - val logger = coroutineContext.logger() - val bucket = attributes[S3Attributes.DirectoryBucket] val client = attributes[S3Attributes.ExpressClient] From 8cf971619f738c83cda3c2443206d4861507a385 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 09:41:29 -0500 Subject: [PATCH 055/145] Change AwsHttpSigner -> HttpSigner --- aws-runtime/aws-http/api/aws-http.api | 4 ++-- .../aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index 88a9dfd857f..e88ff0c3f68 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -104,8 +104,8 @@ public final class aws/sdk/kotlin/runtime/http/OsMetadata { } public final class aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner : aws/smithy/kotlin/runtime/http/auth/HttpSigner { - public fun (Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner;)V - public final fun getAwsHttpSigner ()Laws/smithy/kotlin/runtime/http/auth/AwsHttpSigner; + public fun (Laws/smithy/kotlin/runtime/http/auth/HttpSigner;)V + public final fun getHttpSigner ()Laws/smithy/kotlin/runtime/http/auth/HttpSigner; public fun sign (Laws/smithy/kotlin/runtime/http/auth/SignHttpRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; } diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt index 2f82d315ce4..fd6be26e06a 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt @@ -16,13 +16,13 @@ private const val S3_EXPRESS_SESSION_TOKEN_HEADER = "X-Amz-S3session-Token" private const val SESSION_TOKEN_HEADER = "X-Amz-Security-Token" /** - * An [HttpSigner] used for S3 Express requests. It wraps [AwsHttpSigner] and has identical behavior except for two differences: + * An [HttpSigner] used for S3 Express requests. It has identical behavior with the given [httpSigner] except for two differences: * 1. Adds an `X-Amz-S3Session-Token` header, with a value of the credentials' sessionToken * 2. Removes the `X-Amz-Security-Token` header, which must not be sent for S3 Express requests. - * @param awsHttpSigner An instance of [AwsHttpSigner] + * @param httpSigner An instance of [HttpSigner] */ public class S3ExpressHttpSigner( - public val awsHttpSigner: AwsHttpSigner, + public val httpSigner: HttpSigner, ) : HttpSigner { /** * Sign the request, adding `X-Amz-S3Session-Token` header and removing `X-Amz-Security-Token` header. @@ -39,7 +39,7 @@ public class S3ExpressHttpSigner( mutAttrs[AwsSigningAttributes.OmitSessionToken] = true // 3. call main signer - awsHttpSigner.sign(signingRequest.copy(signingAttributes = mutAttrs)) + httpSigner.sign(signingRequest.copy(signingAttributes = mutAttrs)) // 4. remove session token header signingRequest.httpRequest.headers.remove(SESSION_TOKEN_HEADER) From 1f163605d8c89fc6f1a2d79c531c5b1ac83f4040 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 09:47:05 -0500 Subject: [PATCH 056/145] Remove unnecessary import --- .../src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt index fd6be26e06a..a1252fce3d3 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt @@ -7,7 +7,6 @@ package aws.sdk.kotlin.runtime.http import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAttributes import aws.smithy.kotlin.runtime.collections.toMutableAttributes -import aws.smithy.kotlin.runtime.http.auth.AwsHttpSigner import aws.smithy.kotlin.runtime.http.auth.HttpSigner import aws.smithy.kotlin.runtime.http.auth.SignHttpRequest import aws.smithy.kotlin.runtime.http.request.header From f8d0de5b2911f4bbe3f21003a09e72644525a02a Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 09:52:35 -0500 Subject: [PATCH 057/145] Relocate signer and interceptors to `s3` source set --- aws-runtime/aws-http/api/aws-http.api | 55 ------------------- .../aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt | 2 - .../s3/express/S3ExpressIntegration.kt | 12 +++- .../s3}/S3ExpressCrc32ChecksumInterceptor.kt | 2 +- .../S3ExpressDisableChecksumInterceptor.kt | 2 +- .../services/s3}/S3ExpressHttpSigner.kt | 2 +- .../services/s3/SigV4S3ExpressAuthScheme.kt | 1 - 7 files changed, 13 insertions(+), 63 deletions(-) rename {aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors => services/s3/common/src/aws/sdk/kotlin/services/s3}/S3ExpressCrc32ChecksumInterceptor.kt (97%) rename {aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors => services/s3/common/src/aws/sdk/kotlin/services/s3}/S3ExpressDisableChecksumInterceptor.kt (96%) rename {aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http => services/s3/common/src/aws/sdk/kotlin/services/s3}/S3ExpressHttpSigner.kt (98%) diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index e88ff0c3f68..1be03dc3e03 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -103,12 +103,6 @@ public final class aws/sdk/kotlin/runtime/http/OsMetadata { public fun toString ()Ljava/lang/String; } -public final class aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner : aws/smithy/kotlin/runtime/http/auth/HttpSigner { - public fun (Laws/smithy/kotlin/runtime/http/auth/HttpSigner;)V - public final fun getHttpSigner ()Laws/smithy/kotlin/runtime/http/auth/HttpSigner; - public fun sign (Laws/smithy/kotlin/runtime/http/auth/SignHttpRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; -} - public final class aws/sdk/kotlin/runtime/http/SdkMetadata { public fun (Ljava/lang/String;Ljava/lang/String;)V public final fun component1 ()Ljava/lang/String; @@ -168,55 +162,6 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/AwsSpanInterceptor : public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V } -public final class aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { - public fun ()V - public fun (Ljava/lang/String;)V - public synthetic fun (Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V - public final fun getChecksumAlgorithmHeaderName ()Ljava/lang/String; - public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeRetryLoop (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun readAfterAttempt (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterDeserialization (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterExecution (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterSerialization (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readAfterSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readAfterTransmit (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V - public fun readBeforeAttempt (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V - public fun readBeforeExecution (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V - public fun readBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V - public fun readBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V -} - -public final class aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { - public fun ()V - public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeRetryLoop (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun modifyBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; - public fun readAfterAttempt (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterDeserialization (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterExecution (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;)V - public fun readAfterSerialization (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readAfterSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readAfterTransmit (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V - public fun readBeforeAttempt (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)V - public fun readBeforeExecution (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V - public fun readBeforeSerialization (Laws/smithy/kotlin/runtime/client/RequestInterceptorContext;)V - public fun readBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V - public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V -} - public final class aws/sdk/kotlin/runtime/http/interceptors/UnsupportedSigningAlgorithmInterceptor : aws/smithy/kotlin/runtime/client/Interceptor { public fun ()V public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt index b51acb0cdb1..e36fb00cd69 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/AwsRuntimeTypes.kt @@ -60,8 +60,6 @@ object AwsRuntimeTypes { object Interceptors : RuntimeTypePackage(AwsKotlinDependency.AWS_HTTP, "interceptors") { val AddUserAgentMetadataInterceptor = symbol("AddUserAgentMetadataInterceptor") val UnsupportedSigningAlgorithmInterceptor = symbol("UnsupportedSigningAlgorithmInterceptor") - val S3ExpressCrc32ChecksumInterceptor = symbol("S3ExpressCrc32ChecksumInterceptor") - val S3ExpressDisableChecksumInterceptor = symbol("S3ExpressDisableChecksumInterceptor") } object Retries { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index fd970290825..861ef4e3399 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -113,6 +113,11 @@ class S3ExpressIntegration : KotlinIntegration { !op.isS3UploadPart && (op.hasTrait() || op.hasTrait()) override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { + val interceptorSymbol = buildSymbol { + namespace = "aws.sdk.kotlin.services.s3" + name = "S3ExpressCrc32ChecksumInterceptor" + } + val httpChecksumTrait = op.getTrait() val checksumAlgorithmMember = ctx.model.expectShape(op.input.get()) @@ -125,7 +130,7 @@ class S3ExpressIntegration : KotlinIntegration { val checksumRequired = op.hasTrait() || httpChecksumTrait?.isRequestChecksumRequired == true if (checksumRequired) { - writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq() ?: ""}))", AwsRuntimeTypes.Http.Interceptors.S3ExpressCrc32ChecksumInterceptor) + writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq() ?: ""}))", interceptorSymbol) } } } @@ -136,7 +141,10 @@ class S3ExpressIntegration : KotlinIntegration { override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = op.isS3UploadPart override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val interceptorSymbol = AwsRuntimeTypes.Http.Interceptors.S3ExpressDisableChecksumInterceptor + val interceptorSymbol = buildSymbol { + namespace = "aws.sdk.kotlin.services.s3" + name = "S3ExpressDisableChecksumInterceptor" + } writer.addImport(interceptorSymbol) writer.write("op.interceptors.add(#T())", interceptorSymbol) } diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCrc32ChecksumInterceptor.kt similarity index 97% rename from aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCrc32ChecksumInterceptor.kt index 563345bd11f..aa8ad91fc5d 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressCrc32ChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCrc32ChecksumInterceptor.kt @@ -2,7 +2,7 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.runtime.http.interceptors +package aws.sdk.kotlin.services.s3 import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext import aws.smithy.kotlin.runtime.collections.AttributeKey diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressDisableChecksumInterceptor.kt similarity index 96% rename from aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressDisableChecksumInterceptor.kt index aedc5c1bed0..536b14ca70f 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/S3ExpressDisableChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressDisableChecksumInterceptor.kt @@ -2,7 +2,7 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.runtime.http.interceptors +package aws.sdk.kotlin.services.s3 import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext import aws.smithy.kotlin.runtime.collections.AttributeKey diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt similarity index 98% rename from aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt index a1252fce3d3..4f71ad453ba 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/S3ExpressHttpSigner.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt @@ -2,7 +2,7 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.runtime.http +package aws.sdk.kotlin.services.s3 import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAttributes diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt index bbe12adabae..947d76e0e49 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt @@ -5,7 +5,6 @@ package aws.sdk.kotlin.services.s3 -import aws.sdk.kotlin.runtime.http.S3ExpressHttpSigner import aws.smithy.kotlin.runtime.InternalApi import aws.smithy.kotlin.runtime.auth.AuthOption import aws.smithy.kotlin.runtime.auth.AuthSchemeId From 0df8638eade8609980d05bc2593166f146970f94 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 11:09:02 -0500 Subject: [PATCH 058/145] Don't inherit `SigV4AuthSchemeIntegration` --- .../SigV4S3ExpressAuthSchemeIntegration.kt | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index 93c7697def3..d1a9c7b6826 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -14,16 +14,19 @@ import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter import software.amazon.smithy.kotlin.codegen.integration.AuthSchemeHandler +import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding import software.amazon.smithy.kotlin.codegen.model.buildSymbol import software.amazon.smithy.kotlin.codegen.model.expectShape import software.amazon.smithy.kotlin.codegen.model.hasTrait import software.amazon.smithy.kotlin.codegen.model.knowledge.AwsSignatureVersion4 -import software.amazon.smithy.kotlin.codegen.rendering.auth.SigV4AuthSchemeIntegration +import software.amazon.smithy.kotlin.codegen.rendering.auth.Sigv4SignedBodyHeaderMiddleware +import software.amazon.smithy.kotlin.codegen.rendering.auth.credentialsProviderProp import software.amazon.smithy.kotlin.codegen.rendering.endpoints.EndpointCustomization import software.amazon.smithy.kotlin.codegen.rendering.endpoints.EndpointPropertyRenderer import software.amazon.smithy.kotlin.codegen.rendering.endpoints.ExpressionRenderer import software.amazon.smithy.kotlin.codegen.rendering.protocol.* +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.node.Node @@ -36,9 +39,20 @@ import java.util.* /** * Register support for the `sigv4-s3express` auth scheme. */ -class SigV4S3ExpressAuthSchemeIntegration : SigV4AuthSchemeIntegration() { +class SigV4S3ExpressAuthSchemeIntegration : KotlinIntegration { + override val order: Byte = -50 + override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = model.expectShape(settings.service).isS3 + override fun authSchemes(ctx: ProtocolGenerator.GenerationContext): List = listOf(SigV4S3ExpressAuthSchemeHandler()) + + override fun customizeMiddleware( + ctx: ProtocolGenerator.GenerationContext, + resolved: List, + ): List = resolved + Sigv4SignedBodyHeaderMiddleware() + + override fun additionalServiceConfigProps(ctx: CodegenContext): List = listOf(credentialsProviderProp) + override fun customizeEndpointResolution(ctx: ProtocolGenerator.GenerationContext): EndpointCustomization = SigV4S3ExpressEndpointCustomization override val sectionWriters: List @@ -46,10 +60,8 @@ class SigV4S3ExpressAuthSchemeIntegration : SigV4AuthSchemeIntegration() { // add S3 Express credentials provider to managed resources in the service client initializer private val renderClientInitializer = AppendingSectionWriter { writer -> - writer.write("managedResources.addIfManaged(config.s3ExpressCredentialsProvider)") + writer.write("managedResources.#T(config.s3ExpressCredentialsProvider)", RuntimeTypes.Core.IO.addIfManaged) } - - override fun authSchemes(ctx: ProtocolGenerator.GenerationContext): List = listOf(SigV4S3ExpressAuthSchemeHandler()) } internal val sigV4S3ExpressSymbol = buildSymbol { From 2c0d44aa91ae0057dbbadbf5d40e89ff18a92ef4 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 11:10:51 -0500 Subject: [PATCH 059/145] ktlint --- .../codegen/customization/s3/express/S3ExpressIntegration.kt | 1 - .../s3/express/SigV4S3ExpressAuthSchemeIntegration.kt | 1 - services/s3/e2eTest/src/S3ExpressTest.kt | 1 - 3 files changed, 3 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 861ef4e3399..f5fdb9e2567 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -4,7 +4,6 @@ */ package aws.sdk.kotlin.codegen.customization.s3.express -import aws.sdk.kotlin.codegen.AwsRuntimeTypes import aws.sdk.kotlin.codegen.customization.s3.ClientConfigIntegration import aws.sdk.kotlin.codegen.customization.s3.isS3 import software.amazon.smithy.aws.traits.HttpChecksumTrait diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index d1a9c7b6826..b09913eb65c 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -4,7 +4,6 @@ */ package aws.sdk.kotlin.codegen.customization.s3.express -import aws.sdk.kotlin.codegen.AwsRuntimeTypes import aws.sdk.kotlin.codegen.customization.s3.ClientConfigIntegration import aws.sdk.kotlin.codegen.customization.s3.isS3 import software.amazon.smithy.aws.traits.auth.UnsignedPayloadTrait diff --git a/services/s3/e2eTest/src/S3ExpressTest.kt b/services/s3/e2eTest/src/S3ExpressTest.kt index e05c9e65659..56905c09af3 100644 --- a/services/s3/e2eTest/src/S3ExpressTest.kt +++ b/services/s3/e2eTest/src/S3ExpressTest.kt @@ -9,7 +9,6 @@ import aws.sdk.kotlin.services.s3.model.* import aws.sdk.kotlin.services.s3.putObject import aws.sdk.kotlin.services.s3.withConfig import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext -import aws.smithy.kotlin.runtime.collections.AttributeKey import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.decodeToString import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor From 73a39db4f3aacc0f1178260a3063ddd2a8d3876b Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 11:11:58 -0500 Subject: [PATCH 060/145] Revert move --- ...e.amazon.smithy.kotlin.codegen.integration.KotlinIntegration | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration index 5e227dbb81f..cbcf623fe0b 100644 --- a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +++ b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration @@ -20,12 +20,12 @@ aws.sdk.kotlin.codegen.customization.s3.ClientConfigIntegration aws.sdk.kotlin.codegen.customization.s3.ContinueIntegration aws.sdk.kotlin.codegen.customization.s3.GetObjectResponseLengthValidationIntegration aws.sdk.kotlin.codegen.customization.s3.HttpPathFilter -aws.sdk.kotlin.codegen.customization.s3.S3ErrorWith200StatusIntegration aws.sdk.kotlin.codegen.customization.s3.TruncatablePaginationIntegration aws.sdk.kotlin.codegen.customization.s3.HostPrefixRequestRouteFilter aws.sdk.kotlin.codegen.customization.s3.UnwrappedXmlOutputIntegration aws.sdk.kotlin.codegen.customization.s3control.HostPrefixFilter aws.sdk.kotlin.codegen.customization.s3control.ClientConfigIntegration +aws.sdk.kotlin.codegen.customization.s3.S3ErrorWith200StatusIntegration aws.sdk.kotlin.codegen.customization.apigateway.ApiGatewayAddAcceptHeader aws.sdk.kotlin.codegen.customization.glacier.GlacierAddVersionHeader aws.sdk.kotlin.codegen.customization.glacier.GlacierAccountIdDefault From b308381da4764d01dc329b7ebd61ba02e714ae47 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 14:40:57 -0500 Subject: [PATCH 061/145] Fix API breakage of FlexibleChecksumsRequestInterceptor --- .../flexiblechecksums/FlexibleChecksumsRequest.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 74b0ae0e654..5fb69ebab68 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -44,6 +44,7 @@ class FlexibleChecksumsRequest : KotlinIntegration { override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { val interceptorSymbol = RuntimeTypes.HttpClient.Interceptors.FlexibleChecksumsRequestInterceptor + val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.inputShape)) val httpChecksumTrait = op.getTrait()!! @@ -51,7 +52,7 @@ class FlexibleChecksumsRequest : KotlinIntegration { .members() .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } - writer.write("op.interceptors.add(#T())", interceptorSymbol) + writer.write("op.interceptors.add(#T<#T>())", interceptorSymbol, inputSymbol) writer.withBlock("input.${requestAlgorithmMember.defaultName()}?.let {", "}") { writer.write("op.context[#T.ChecksumAlgorithm] = it.value", RuntimeTypes.HttpClient.Operation.HttpOperationContext) } From 5e49bde02bd03d62e99c47ac7e0c5299365f910b Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 16:09:22 -0500 Subject: [PATCH 062/145] revert visibility changes --- .../s3/express/SigV4S3ExpressAuthSchemeIntegration.kt | 7 ------- 1 file changed, 7 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index b09913eb65c..9b61775fe11 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -45,13 +45,6 @@ class SigV4S3ExpressAuthSchemeIntegration : KotlinIntegration { override fun authSchemes(ctx: ProtocolGenerator.GenerationContext): List = listOf(SigV4S3ExpressAuthSchemeHandler()) - override fun customizeMiddleware( - ctx: ProtocolGenerator.GenerationContext, - resolved: List, - ): List = resolved + Sigv4SignedBodyHeaderMiddleware() - - override fun additionalServiceConfigProps(ctx: CodegenContext): List = listOf(credentialsProviderProp) - override fun customizeEndpointResolution(ctx: ProtocolGenerator.GenerationContext): EndpointCustomization = SigV4S3ExpressEndpointCustomization override val sectionWriters: List From 01f0ca86951ae712d409d527a25666172a1e71b1 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 16:11:50 -0500 Subject: [PATCH 063/145] remove unused imports --- .../s3/express/SigV4S3ExpressAuthSchemeIntegration.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index 9b61775fe11..71fc74dbb9c 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -19,13 +19,10 @@ import software.amazon.smithy.kotlin.codegen.model.buildSymbol import software.amazon.smithy.kotlin.codegen.model.expectShape import software.amazon.smithy.kotlin.codegen.model.hasTrait import software.amazon.smithy.kotlin.codegen.model.knowledge.AwsSignatureVersion4 -import software.amazon.smithy.kotlin.codegen.rendering.auth.Sigv4SignedBodyHeaderMiddleware -import software.amazon.smithy.kotlin.codegen.rendering.auth.credentialsProviderProp import software.amazon.smithy.kotlin.codegen.rendering.endpoints.EndpointCustomization import software.amazon.smithy.kotlin.codegen.rendering.endpoints.EndpointPropertyRenderer import software.amazon.smithy.kotlin.codegen.rendering.endpoints.ExpressionRenderer import software.amazon.smithy.kotlin.codegen.rendering.protocol.* -import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model import software.amazon.smithy.model.node.Node From e7157c824983c195fbcbfa73f4ace744a2c11968 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 16:20:33 -0500 Subject: [PATCH 064/145] update providerName --- .../kotlin/services/s3/internal/S3ExpressCredentialsCache.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index fbb66f56062..4efa1c3b095 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -96,7 +96,7 @@ internal class S3ExpressCredentialsCache( secretAccessKey = credentials.secretAccessKey, sessionToken = credentials.sessionToken, expiration = credentials.expiration, - providerName = "S3ExpressCredentialsProvider", + providerName = "DefaultS3ExpressCredentialsProvider", ), credentials.expiration, ) From e9b8d4c6b9bec4f7df87b050ab3e7329cee61e97 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 16:51:36 -0500 Subject: [PATCH 065/145] remove `client` from `S3ExpressCredentialsCacheKey` --- .../s3/internal/S3ExpressCredentialsCache.kt | 10 +++++----- .../s3/internal/SdkS3ExpressCredentialsProvider.kt | 13 +++++++------ .../s3/internal/S3ExpressCredentialsCacheTest.kt | 7 +++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 4efa1c3b095..8bb7b62b535 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -25,6 +25,7 @@ private val REFRESH_BUFFER = 1.minutes private val DEFAULT_REFRESH_PERIOD = 5.minutes internal class S3ExpressCredentialsCache( + private val client: S3Client, private val clock: Clock = Clock.System, ) : CoroutineScope, Closeable { override val coroutineContext: CoroutineContext = Job() + CoroutineName("S3ExpressCredentialsCacheRefresh") @@ -39,7 +40,7 @@ internal class S3ExpressCredentialsCache( } suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru.get(key)?.takeIf { !it.isExpired(clock) }?.value - ?: (createSessionCredentials(key).also { put(key, it) }).value + ?: (createSessionCredentials(key.bucket).also { put(key, it) }).value suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue) { lru.put(key, value) @@ -66,7 +67,7 @@ internal class S3ExpressCredentialsCache( if (cachedValue.isExpired(clock)) { logger.debug { "Credentials for ${key.bucket} expire within the refresh buffer period, performing a refresh..." } - createSessionCredentials(key).also { + createSessionCredentials(key.bucket).also { refreshedCredentials[key] = it nextRefresh = minOf(nextRefresh, it.expiresAt - REFRESH_BUFFER) } @@ -87,8 +88,8 @@ internal class S3ExpressCredentialsCache( } } - private suspend fun createSessionCredentials(key: S3ExpressCredentialsCacheKey): ExpiringValue { - val credentials = (key.client as S3Client).createSession { bucket = key.bucket }.credentials!! + private suspend fun createSessionCredentials(bucket: String): ExpiringValue { + val credentials = client.createSession { this.bucket = bucket }.credentials!! return ExpiringValue( Credentials( @@ -110,6 +111,5 @@ internal class S3ExpressCredentialsCache( internal data class S3ExpressCredentialsCacheKey( val bucket: String, - val client: SdkClient, val credentials: Credentials, ) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt index 495faed85ea..7fbeb82f87e 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt @@ -5,6 +5,7 @@ package aws.sdk.kotlin.services.s3.internal import aws.sdk.kotlin.services.s3.S3Attributes +import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.S3ExpressCredentialsProvider import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider @@ -13,16 +14,16 @@ import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.io.SdkManagedBase internal class DefaultS3ExpressCredentialsProvider( - val bootstrapCredentialsProvider: CredentialsProvider, + val bootstrapCredentialsProvider: CredentialsProvider ) : S3ExpressCredentialsProvider, SdkManagedBase() { - private val credentialsCache = S3ExpressCredentialsCache() + private lateinit var credentialsCache: S3ExpressCredentialsCache override suspend fun resolve(attributes: Attributes): Credentials { - val bucket = attributes[S3Attributes.DirectoryBucket] - val client = attributes[S3Attributes.ExpressClient] - - val key = S3ExpressCredentialsCacheKey(bucket, client, bootstrapCredentialsProvider.resolve(attributes)) + if (!this::credentialsCache.isInitialized) { + credentialsCache = S3ExpressCredentialsCache(attributes[S3Attributes.ExpressClient] as S3Client) + } + val key = S3ExpressCredentialsCacheKey(attributes[S3Attributes.DirectoryBucket], bootstrapCredentialsProvider.resolve(attributes)) return credentialsCache.get(key) } diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt index 840dee9c5aa..fd6eb85560c 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt @@ -14,12 +14,11 @@ public class S3ExpressCredentialsCacheTest { @Test fun testCacheKeyEquality() = runTest { val bucket = "bucket" - val client = S3Client.builder().build() val testCredentials = Credentials("accessKeyId", "secretAccessKey", "sessionToken") - // Different keys with the same bucket, client, and credentials should be considered equal - val key1 = S3ExpressCredentialsCacheKey(bucket, client, testCredentials) - val key2 = S3ExpressCredentialsCacheKey(bucket, client, testCredentials) + // Different keys with the same bucket and credentials should be considered equal + val key1 = S3ExpressCredentialsCacheKey(bucket, testCredentials) + val key2 = S3ExpressCredentialsCacheKey(bucket, testCredentials) assertEquals(key1, key2) } From 0fd4f669fbc5dbefffa016f9142bb84e15842ee9 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 14 Feb 2024 17:01:21 -0500 Subject: [PATCH 066/145] Make `S3ExpressHttpSigner` internal --- .../src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt index 4f71ad453ba..e097161ad9a 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt @@ -20,13 +20,13 @@ private const val SESSION_TOKEN_HEADER = "X-Amz-Security-Token" * 2. Removes the `X-Amz-Security-Token` header, which must not be sent for S3 Express requests. * @param httpSigner An instance of [HttpSigner] */ -public class S3ExpressHttpSigner( - public val httpSigner: HttpSigner, +internal class S3ExpressHttpSigner( + private val httpSigner: HttpSigner, ) : HttpSigner { /** * Sign the request, adding `X-Amz-S3Session-Token` header and removing `X-Amz-Security-Token` header. */ - public override suspend fun sign(signingRequest: SignHttpRequest) { + override suspend fun sign(signingRequest: SignHttpRequest) { val sessionToken = (signingRequest.identity as? Credentials)?.sessionToken ?: error("Failed to parse sessionToken from identity") From 6b15f3c6e13f7bb2a7e602252be4d184730026ee Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 15 Feb 2024 10:53:52 -0500 Subject: [PATCH 067/145] Simplify refresh logic --- .../s3/internal/S3ExpressCredentialsCache.kt | 48 ++++++++----------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 8bb7b62b535..51f6987ba4d 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -19,6 +19,7 @@ import kotlinx.coroutines.channels.* import kotlinx.coroutines.selects.* import kotlin.coroutines.CoroutineContext import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 private val REFRESH_BUFFER = 1.minutes @@ -31,7 +32,6 @@ internal class S3ExpressCredentialsCache( override val coroutineContext: CoroutineContext = Job() + CoroutineName("S3ExpressCredentialsCacheRefresh") private val lru = LruCache>(DEFAULT_S3_EXPRESS_CACHE_SIZE) - private val immediateRefreshChannel = Channel(Channel.CONFLATED) // channel used to indicate an immediate refresh attempt is required init { launch(coroutineContext) { @@ -44,10 +44,9 @@ internal class S3ExpressCredentialsCache( suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue) { lru.put(key, value) - immediateRefreshChannel.send(Unit) } - private fun ExpiringValue.isExpired(clock: Clock): Boolean = clock.now().until(expiresAt).absoluteValue <= REFRESH_BUFFER + private fun ExpiringValue.isExpired(clock: Clock): Boolean = clock.now().until(expiresAt).absoluteValue - 5.minutes <= REFRESH_BUFFER /** * Attempt to refresh the credentials in the cache. A refresh is initiated when: @@ -55,35 +54,29 @@ internal class S3ExpressCredentialsCache( * * the `nextRefresh` time has been reached, which is either `DEFAULT_REFRESH_PERIOD` or * the soonest credentials expiration time (minus a buffer), whichever comes first. */ + @OptIn(ExperimentalCoroutinesApi::class) private suspend fun refresh() { - val logger = coroutineContext.logger() while (isActive) { - val refreshedCredentials = mutableMapOf>() - var nextRefresh: Instant = clock.now() + DEFAULT_REFRESH_PERIOD - - lru.withLock { - lru.entries.forEach { (key, cachedValue) -> - nextRefresh = minOf(nextRefresh, cachedValue.expiresAt - REFRESH_BUFFER) - - if (cachedValue.isExpired(clock)) { - logger.debug { "Credentials for ${key.bucket} expire within the refresh buffer period, performing a refresh..." } - createSessionCredentials(key.bucket).also { - refreshedCredentials[key] = it - nextRefresh = minOf(nextRefresh, it.expiresAt - REFRESH_BUFFER) - } - } - } - - refreshedCredentials.forEach { (key, value) -> - lru.remove(key) - lru.putUnlocked(key, value) - } + if (lru.size == 0) { + println("CACHE: LRU is empty, continuing...") + delay(5.seconds) + continue } - // wake up when it's time to refresh or an immediate refresh has been triggered - select { + // Refresh any credentials which are already expired + val expiredEntries = lru.entries.filter { it.value.isExpired(clock) } + println("CACHE: ${expiredEntries.size} entries are expired, refreshing...") + expiredEntries.forEach { entry -> + lru.put(entry.key, createSessionCredentials(entry.key.bucket)) + } + + // Find the next expiring credentials, sleep until then + val nextExpiringEntry = lru.entries.minBy { it.value.expiresAt } + println("CACHE: Current time ${clock.now()}, next expiration: ${nextExpiringEntry.value.expiresAt} ") + val nextRefresh = minOf(clock.now() + DEFAULT_REFRESH_PERIOD, nextExpiringEntry.value.expiresAt - REFRESH_BUFFER) + + select { onTimeout(clock.now().until(nextRefresh)) {} - immediateRefreshChannel.onReceive {} } } } @@ -105,7 +98,6 @@ internal class S3ExpressCredentialsCache( override fun close() { coroutineContext.cancel(null) - immediateRefreshChannel.close() } } From 993690db1445b42b7f339496f3ff787e8cf96b8e Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 15 Feb 2024 12:11:01 -0500 Subject: [PATCH 068/145] Simplify refresh logic --- .../s3/internal/S3ExpressCredentialsCache.kt | 52 ++++++++++++++----- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 51f6987ba4d..1ecb568ac69 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -9,6 +9,7 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.collections.LruCache import aws.smithy.kotlin.runtime.io.Closeable +import aws.smithy.kotlin.runtime.telemetry.TelemetryProviderContext import aws.smithy.kotlin.runtime.telemetry.logging.logger import aws.smithy.kotlin.runtime.time.Clock import aws.smithy.kotlin.runtime.time.Instant @@ -31,7 +32,7 @@ internal class S3ExpressCredentialsCache( ) : CoroutineScope, Closeable { override val coroutineContext: CoroutineContext = Job() + CoroutineName("S3ExpressCredentialsCacheRefresh") - private val lru = LruCache>(DEFAULT_S3_EXPRESS_CACHE_SIZE) + private val lru = LruCache(DEFAULT_S3_EXPRESS_CACHE_SIZE) init { launch(coroutineContext) { @@ -39,11 +40,17 @@ internal class S3ExpressCredentialsCache( } } - suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru.get(key)?.takeIf { !it.isExpired(clock) }?.value + suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru + .get(key) + ?.takeIf { !it.expiringCredentials.isExpired(clock) } + ?.let { + it.usedSinceLastRefresh = true + it.expiringCredentials.value + } ?: (createSessionCredentials(key.bucket).also { put(key, it) }).value suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue) { - lru.put(key, value) + lru.put(key, S3ExpressCredentialsCacheValue(value, usedSinceLastRefresh = true)) } private fun ExpiringValue.isExpired(clock: Clock): Boolean = clock.now().until(expiresAt).absoluteValue - 5.minutes <= REFRESH_BUFFER @@ -54,30 +61,42 @@ internal class S3ExpressCredentialsCache( * * the `nextRefresh` time has been reached, which is either `DEFAULT_REFRESH_PERIOD` or * the soonest credentials expiration time (minus a buffer), whichever comes first. */ - @OptIn(ExperimentalCoroutinesApi::class) private suspend fun refresh() { + val logger = coroutineContext.logger() while (isActive) { if (lru.size == 0) { - println("CACHE: LRU is empty, continuing...") + logger.debug { "Cache is empty, waiting..." } delay(5.seconds) continue } + // Evict any credentials that weren't used since the last refresh + lru.entries.filter { !it.value.usedSinceLastRefresh }.forEach { + logger.debug { "Credentials for ${it.key.bucket} were not used since last refresh, evicting..."} + lru.remove(it.key) + } + + // Mark all credentials as not used since last refresh + lru.entries.forEach { + it.value.usedSinceLastRefresh = false + } + // Refresh any credentials which are already expired - val expiredEntries = lru.entries.filter { it.value.isExpired(clock) } - println("CACHE: ${expiredEntries.size} entries are expired, refreshing...") + val expiredEntries = lru.entries.filter { it.value.expiringCredentials.isExpired(clock) } expiredEntries.forEach { entry -> - lru.put(entry.key, createSessionCredentials(entry.key.bucket)) + logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..."} + lru.put(entry.key, S3ExpressCredentialsCacheValue(createSessionCredentials(entry.key.bucket), false)) } // Find the next expiring credentials, sleep until then - val nextExpiringEntry = lru.entries.minBy { it.value.expiresAt } - println("CACHE: Current time ${clock.now()}, next expiration: ${nextExpiringEntry.value.expiresAt} ") - val nextRefresh = minOf(clock.now() + DEFAULT_REFRESH_PERIOD, nextExpiringEntry.value.expiresAt - REFRESH_BUFFER) + val nextExpiringEntry = lru.entries.minByOrNull { it.value.expiringCredentials.expiresAt } - select { - onTimeout(clock.now().until(nextRefresh)) {} - } + val nextRefresh = nextExpiringEntry?.let { + minOf(clock.now() + DEFAULT_REFRESH_PERIOD, it.value.expiringCredentials.expiresAt - REFRESH_BUFFER) + } ?: (clock.now() + DEFAULT_REFRESH_PERIOD) + + logger.debug { "Completed credentials refresh, next attempt in ${clock.now().until(nextRefresh)}" } + delay(clock.now().until(nextRefresh)) } } @@ -105,3 +124,8 @@ internal data class S3ExpressCredentialsCacheKey( val bucket: String, val credentials: Credentials, ) + +internal data class S3ExpressCredentialsCacheValue( + val expiringCredentials: ExpiringValue, + var usedSinceLastRefresh: Boolean +) From b38ecfc4831a0994ba259acb74845423c2706ed8 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 15 Feb 2024 12:11:59 -0500 Subject: [PATCH 069/145] ktlint --- .../services/s3/internal/S3ExpressCredentialsCache.kt | 11 +++-------- .../s3/internal/SdkS3ExpressCredentialsProvider.kt | 2 +- .../s3/internal/S3ExpressCredentialsCacheTest.kt | 1 - 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 1ecb568ac69..d865864483f 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -6,18 +6,13 @@ package aws.sdk.kotlin.services.s3.internal import aws.sdk.kotlin.services.s3.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.collections.LruCache import aws.smithy.kotlin.runtime.io.Closeable -import aws.smithy.kotlin.runtime.telemetry.TelemetryProviderContext import aws.smithy.kotlin.runtime.telemetry.logging.logger import aws.smithy.kotlin.runtime.time.Clock -import aws.smithy.kotlin.runtime.time.Instant import aws.smithy.kotlin.runtime.time.until import aws.smithy.kotlin.runtime.util.ExpiringValue import kotlinx.coroutines.* -import kotlinx.coroutines.channels.* -import kotlinx.coroutines.selects.* import kotlin.coroutines.CoroutineContext import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds @@ -72,7 +67,7 @@ internal class S3ExpressCredentialsCache( // Evict any credentials that weren't used since the last refresh lru.entries.filter { !it.value.usedSinceLastRefresh }.forEach { - logger.debug { "Credentials for ${it.key.bucket} were not used since last refresh, evicting..."} + logger.debug { "Credentials for ${it.key.bucket} were not used since last refresh, evicting..." } lru.remove(it.key) } @@ -84,7 +79,7 @@ internal class S3ExpressCredentialsCache( // Refresh any credentials which are already expired val expiredEntries = lru.entries.filter { it.value.expiringCredentials.isExpired(clock) } expiredEntries.forEach { entry -> - logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..."} + logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..." } lru.put(entry.key, S3ExpressCredentialsCacheValue(createSessionCredentials(entry.key.bucket), false)) } @@ -127,5 +122,5 @@ internal data class S3ExpressCredentialsCacheKey( internal data class S3ExpressCredentialsCacheValue( val expiringCredentials: ExpiringValue, - var usedSinceLastRefresh: Boolean + var usedSinceLastRefresh: Boolean, ) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt index 7fbeb82f87e..026ff3d95b2 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt @@ -14,7 +14,7 @@ import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.io.SdkManagedBase internal class DefaultS3ExpressCredentialsProvider( - val bootstrapCredentialsProvider: CredentialsProvider + val bootstrapCredentialsProvider: CredentialsProvider, ) : S3ExpressCredentialsProvider, SdkManagedBase() { private lateinit var credentialsCache: S3ExpressCredentialsCache diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt index fd6eb85560c..615f793b5ef 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt @@ -4,7 +4,6 @@ */ package aws.sdk.kotlin.services.s3.internal -import aws.sdk.kotlin.services.s3.S3Client import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import kotlinx.coroutines.test.runTest import kotlin.test.Test From 16d8dfdc234147e4b59733c6bb3830e2a9d64363 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 15 Feb 2024 12:16:56 -0500 Subject: [PATCH 070/145] changelog --- .changes/6aef179b-d710-40a5-bd6e-37078f07dfa4.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changes/6aef179b-d710-40a5-bd6e-37078f07dfa4.json diff --git a/.changes/6aef179b-d710-40a5-bd6e-37078f07dfa4.json b/.changes/6aef179b-d710-40a5-bd6e-37078f07dfa4.json new file mode 100644 index 00000000000..68c24754d0d --- /dev/null +++ b/.changes/6aef179b-d710-40a5-bd6e-37078f07dfa4.json @@ -0,0 +1,5 @@ +{ + "id": "6aef179b-d710-40a5-bd6e-37078f07dfa4", + "type": "feature", + "description": "Add support for S3 Express One Zone" +} \ No newline at end of file From 8921174e04ec5f6febfcce827cc545a492635655 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 15 Feb 2024 12:18:45 -0500 Subject: [PATCH 071/145] Update KDocs --- .../services/s3/internal/S3ExpressCredentialsCache.kt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index d865864483f..4ae02a6f092 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -48,13 +48,11 @@ internal class S3ExpressCredentialsCache( lru.put(key, S3ExpressCredentialsCacheValue(value, usedSinceLastRefresh = true)) } - private fun ExpiringValue.isExpired(clock: Clock): Boolean = clock.now().until(expiresAt).absoluteValue - 5.minutes <= REFRESH_BUFFER + private fun ExpiringValue.isExpired(clock: Clock): Boolean = clock.now().until(expiresAt).absoluteValue <= REFRESH_BUFFER /** - * Attempt to refresh the credentials in the cache. A refresh is initiated when: - * * a new set of credentials are added to the cache (immediate refresh) - * * the `nextRefresh` time has been reached, which is either `DEFAULT_REFRESH_PERIOD` or - * the soonest credentials expiration time (minus a buffer), whichever comes first. + * Attempt to refresh the credentials in the cache. A refresh is initiated when the `nextRefresh` time has been reached, + * which is either `DEFAULT_REFRESH_PERIOD` or the soonest credentials expiration time (minus a buffer), whichever comes first. */ private suspend fun refresh() { val logger = coroutineContext.logger() From 7aa8d74ee35ebdaaf811721cf26f6ee0f3947134 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 15 Feb 2024 13:06:46 -0500 Subject: [PATCH 072/145] Use `writer` implicitly --- .../s3/express/SigV4S3ExpressAuthSchemeIntegration.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index 71fc74dbb9c..cd082aac7cc 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -121,8 +121,8 @@ private fun renderAuthSchemes(writer: KotlinWriter, authSchemes: Expression, exp writeInline("disableDoubleUriEncode = ") renderOrElse(expressionRenderer, scheme.getBooleanMember("disableDoubleEncoding"), "false") - writer.writeInline("signingRegion = ") - writer.renderOrElse(expressionRenderer, scheme.getStringMember("signingRegion"), "null") + writeInline("signingRegion = ") + renderOrElse(expressionRenderer, scheme.getStringMember("signingRegion"), "null") } } } From e15051923dbc86058fc5737951b6743761e6806e Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 15 Feb 2024 14:47:07 -0500 Subject: [PATCH 073/145] Correct unit --- .../kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt index 6fac3f2fff4..943503e3844 100644 --- a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt +++ b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt @@ -21,7 +21,7 @@ class S3ExpressBenchmark : ServiceBenchmark { "--$regionAz--x-s3" private val KEY = "64kb-object" - private val CONTENTS = "a".repeat(65536) // 64MB + private val CONTENTS = "a".repeat(65536) // 64KB @OptIn(ExperimentalApi::class) override suspend fun client() = S3Client.fromEnvironment { From e5047920d5758e49f6ddae6bd2b2350f21322fd8 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Fri, 16 Feb 2024 10:20:35 -0500 Subject: [PATCH 074/145] Add a FIXME to use IMDS --- .../kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt index 943503e3844..f87f3684b42 100644 --- a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt +++ b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt @@ -15,7 +15,7 @@ import aws.smithy.kotlin.runtime.content.ByteStream * Note: This benchmark must be run from an EC2 host in the same AZ as the bucket (usw2-az1). */ class S3ExpressBenchmark : ServiceBenchmark { - private val regionAz = "usw2-az1" + private val regionAz = "usw2-az1" // FIXME Use IMDS to dynamically create a bucket in the EC2 host's AZ private val bucketName = Common.random("sdk-benchmark-bucket-") .substring(0 until 47) + // truncate to prevent "bucket name too long" errors "--$regionAz--x-s3" From 52b8acccede94d52322bf71f09a697c18b011ecc Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 13:07:28 -0500 Subject: [PATCH 075/145] Add KDocs --- .../s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt index fc45123ea1d..23cc555605f 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt @@ -7,6 +7,9 @@ package aws.sdk.kotlin.services.s3 import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.collections.AttributeKey +/** + * Execution context attributes specific to S3 + */ public object S3Attributes { /** * The name of the directory bucket requests are being made to From 8bb96054140c5d9dd9b32ea207ad8226b91b890e Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 13:08:57 -0500 Subject: [PATCH 076/145] Clarify error message --- .../src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt index e097161ad9a..69abcdc0f46 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt @@ -28,7 +28,7 @@ internal class S3ExpressHttpSigner( */ override suspend fun sign(signingRequest: SignHttpRequest) { val sessionToken = (signingRequest.identity as? Credentials)?.sessionToken - ?: error("Failed to parse sessionToken from identity") + ?: error("No session token found on identity, required for S3 Express") // 1. add the S3 Express Session Token header signingRequest.httpRequest.header(S3_EXPRESS_SESSION_TOKEN_HEADER, sessionToken) From 723dfadbaa554187d6c328928dcaf421743e57bf Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 13:25:44 -0500 Subject: [PATCH 077/145] Don't overrride functions and add KDocs for `default` companion object function --- .../kotlin/services/s3/S3ExpressCredentialsProvider.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt index 80ea856fb0b..2cdbc139305 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt @@ -14,10 +14,12 @@ import aws.smithy.kotlin.runtime.collections.Attributes * A credentials provider used for making requests to S3 Express One Zone directory buckets. */ public interface S3ExpressCredentialsProvider : CloseableCredentialsProvider { - override suspend fun resolve(attributes: Attributes): Credentials - override fun close() - public companion object { + /** + * Create an instance of the default [S3ExpressCredentialsProvider] implementation + * @param bootstrapCredentialsProvider the credentials provider used to call s3:CreateSession to retrieve S3 Express + * credentials + */ public fun default(bootstrapCredentialsProvider: CredentialsProvider): S3ExpressCredentialsProvider = DefaultS3ExpressCredentialsProvider(bootstrapCredentialsProvider) } From 4bb777726d65cc4f2e7b405128c2ea5cabaf4610 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 13:26:43 -0500 Subject: [PATCH 078/145] Clean up config property --- .../s3/ClientConfigIntegration.kt | 18 +++++------------- .../SigV4S3ExpressAuthSchemeIntegration.kt | 4 ++-- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index c52f17f9448..6a20dae0edc 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -71,8 +71,8 @@ class ClientConfigIntegration : KotlinIntegration { """.trimIndent() } - val S3ExpressCredentialsProvider: ConfigProperty = ConfigProperty { - name = "s3ExpressCredentialsProvider" + val ExpressCredentialsProvider: ConfigProperty = ConfigProperty { + name = "expressCredentialsProvider" symbol = buildSymbol { name = "S3ExpressCredentialsProvider" nullable = false @@ -82,21 +82,13 @@ class ClientConfigIntegration : KotlinIntegration { Credentials provider to be used for making requests to S3 Express. """.trimIndent() - additionalImports = listOf( - buildSymbol { - name = "S3ExpressCredentialsProvider" - nullable = false - namespace = "aws.sdk.kotlin.services.s3" - }, - ) - propertyType = ConfigPropertyType.Custom( render = { _, writer -> - writer.write("public val $name: S3ExpressCredentialsProvider = builder.$name ?: S3ExpressCredentialsProvider.default(this.credentialsProvider)") + writer.write("public val #1L: #2T = builder.#1L ?: #2T.default(this.credentialsProvider)", name, symbol) }, renderBuilder = { prop, writer -> prop.documentation?.let(writer::dokka) - writer.write("public var $name: S3ExpressCredentialsProvider? = null") + writer.write("public var #L: #T? = null", name, symbol) }, ) } @@ -126,7 +118,7 @@ class ClientConfigIntegration : KotlinIntegration { UseArnRegionProp, DisableMrapProp, DisableExpressSessionAuth, - S3ExpressCredentialsProvider, + ExpressCredentialsProvider, ) override val sectionWriters: List diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index cd082aac7cc..ea9da6d3da5 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -49,7 +49,7 @@ class SigV4S3ExpressAuthSchemeIntegration : KotlinIntegration { // add S3 Express credentials provider to managed resources in the service client initializer private val renderClientInitializer = AppendingSectionWriter { writer -> - writer.write("managedResources.#T(config.s3ExpressCredentialsProvider)", RuntimeTypes.Core.IO.addIfManaged) + writer.write("managedResources.#T(config.expressCredentialsProvider)", RuntimeTypes.Core.IO.addIfManaged) } } @@ -81,7 +81,7 @@ open class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { } override fun identityProviderAdapterExpression(writer: KotlinWriter) { - writer.write("config.#T", ClientConfigIntegration.S3ExpressCredentialsProvider.propertyName) + writer.write("config.#T", ClientConfigIntegration.ExpressCredentialsProvider.propertyName) } override fun authSchemeProviderInstantiateAuthOptionExpr( From 56ba19217e21c827f35850480b6d6a7230d3bcd9 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 14:23:26 -0500 Subject: [PATCH 079/145] Relax to `httpSigner` --- .../aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt index 947d76e0e49..cc2d1ffbda4 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt @@ -22,7 +22,7 @@ public val AuthSchemeId.Companion.AwsSigV4S3Express: AuthSchemeId * HTTP auth scheme for S3 Express One Zone authentication */ public class SigV4S3ExpressAuthScheme( - awsHttpSigner: AwsHttpSigner, + httpSigner: HttpSigner, ) : AuthScheme { public constructor(awsSigner: AwsSigner, serviceName: String? = null) : this( AwsHttpSigner( @@ -35,7 +35,7 @@ public class SigV4S3ExpressAuthScheme( ) override val schemeId: AuthSchemeId = AuthSchemeId.AwsSigV4S3Express - override val signer: HttpSigner = S3ExpressHttpSigner(awsHttpSigner) + override val signer: HttpSigner = S3ExpressHttpSigner(httpSigner) } /** From 829c48be45b39b873c7608762169618be54d03ac Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 14:24:08 -0500 Subject: [PATCH 080/145] Rename file to match class --- ...dentialsProvider.kt => DefaultS3ExpressCredentialsProvider.kt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename services/s3/common/src/aws/sdk/kotlin/services/s3/internal/{SdkS3ExpressCredentialsProvider.kt => DefaultS3ExpressCredentialsProvider.kt} (100%) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/DefaultS3ExpressCredentialsProvider.kt similarity index 100% rename from services/s3/common/src/aws/sdk/kotlin/services/s3/internal/SdkS3ExpressCredentialsProvider.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/internal/DefaultS3ExpressCredentialsProvider.kt From c54019043a0685d6405284bca4231a11cd18c137 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 14:32:40 -0500 Subject: [PATCH 081/145] Fix config symbol --- .../s3/express/SigV4S3ExpressAuthSchemeIntegration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index ea9da6d3da5..3b8c8fb795c 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -81,7 +81,7 @@ open class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { } override fun identityProviderAdapterExpression(writer: KotlinWriter) { - writer.write("config.#T", ClientConfigIntegration.ExpressCredentialsProvider.propertyName) + writer.write("config.#L", ClientConfigIntegration.ExpressCredentialsProvider.propertyName) } override fun authSchemeProviderInstantiateAuthOptionExpr( From 24bcb746e29f7f9efc32aec284c62bf6fdb76dd9 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 14:34:00 -0500 Subject: [PATCH 082/145] Grab a local copy of `entries` instead of recomputing it --- .../services/s3/internal/S3ExpressCredentialsCache.kt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 4ae02a6f092..72b0e96d6c9 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -63,26 +63,28 @@ internal class S3ExpressCredentialsCache( continue } + val entries = lru.entries + // Evict any credentials that weren't used since the last refresh - lru.entries.filter { !it.value.usedSinceLastRefresh }.forEach { + entries.filter { !it.value.usedSinceLastRefresh }.forEach { logger.debug { "Credentials for ${it.key.bucket} were not used since last refresh, evicting..." } lru.remove(it.key) } // Mark all credentials as not used since last refresh - lru.entries.forEach { + entries.forEach { it.value.usedSinceLastRefresh = false } // Refresh any credentials which are already expired - val expiredEntries = lru.entries.filter { it.value.expiringCredentials.isExpired(clock) } + val expiredEntries = entries.filter { it.value.expiringCredentials.isExpired(clock) } expiredEntries.forEach { entry -> logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..." } lru.put(entry.key, S3ExpressCredentialsCacheValue(createSessionCredentials(entry.key.bucket), false)) } // Find the next expiring credentials, sleep until then - val nextExpiringEntry = lru.entries.minByOrNull { it.value.expiringCredentials.expiresAt } + val nextExpiringEntry = entries.minByOrNull { it.value.expiringCredentials.expiresAt } val nextRefresh = nextExpiringEntry?.let { minOf(clock.now() + DEFAULT_REFRESH_PERIOD, it.value.expiringCredentials.expiresAt - REFRESH_BUFFER) From 88bcc038a51f153448bade7877a123fe5dcf1edc Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 14:47:36 -0500 Subject: [PATCH 083/145] DirectoryBucket -> Bucket --- .../s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt | 4 ++-- .../s3/internal/DefaultS3ExpressCredentialsProvider.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt index 23cc555605f..053b121b5ac 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt @@ -12,9 +12,9 @@ import aws.smithy.kotlin.runtime.collections.AttributeKey */ public object S3Attributes { /** - * The name of the directory bucket requests are being made to + * The name of the bucket requests are being made to */ - public val DirectoryBucket: AttributeKey = AttributeKey("aws.sdk.kotlin#S3ExpressBucket") + public val Bucket: AttributeKey = AttributeKey("aws.sdk.kotlin#S3Bucket") /** * The S3 client being used to make requests to directory buckets. This client will be used to call s3:CreateSession diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/DefaultS3ExpressCredentialsProvider.kt index 026ff3d95b2..a0ec134e313 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/DefaultS3ExpressCredentialsProvider.kt @@ -23,7 +23,7 @@ internal class DefaultS3ExpressCredentialsProvider( credentialsCache = S3ExpressCredentialsCache(attributes[S3Attributes.ExpressClient] as S3Client) } - val key = S3ExpressCredentialsCacheKey(attributes[S3Attributes.DirectoryBucket], bootstrapCredentialsProvider.resolve(attributes)) + val key = S3ExpressCredentialsCacheKey(attributes[S3Attributes.Bucket], bootstrapCredentialsProvider.resolve(attributes)) return credentialsCache.get(key) } From 1227f903c4a4afed1178a1918eec4a8b7a9ec08e Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 14:47:44 -0500 Subject: [PATCH 084/145] logger.trace --- .../kotlin/services/s3/internal/S3ExpressCredentialsCache.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt index 72b0e96d6c9..410deb3103d 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt @@ -58,7 +58,7 @@ internal class S3ExpressCredentialsCache( val logger = coroutineContext.logger() while (isActive) { if (lru.size == 0) { - logger.debug { "Cache is empty, waiting..." } + logger.trace { "Cache is empty, waiting..." } delay(5.seconds) continue } From ca038d6ccf8a6fd734f5e3a1591b307bad687680 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 15:17:08 -0500 Subject: [PATCH 085/145] Relocate things to `express` package --- .../kotlin/codegen/customization/s3/ClientConfigIntegration.kt | 2 +- .../src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt | 3 ++- .../DefaultS3ExpressCredentialsProvider.kt | 3 +-- .../s3/{ => express}/S3ExpressCrc32ChecksumInterceptor.kt | 2 +- .../s3/{internal => express}/S3ExpressCredentialsCache.kt | 2 +- .../services/s3/{ => express}/S3ExpressCredentialsProvider.kt | 3 +-- .../s3/{ => express}/S3ExpressDisableChecksumInterceptor.kt | 2 +- .../kotlin/services/s3/{ => express}/S3ExpressHttpSigner.kt | 2 +- .../s3/{internal => express}/S3ExpressCredentialsCacheTest.kt | 2 +- 9 files changed, 10 insertions(+), 11 deletions(-) rename services/s3/common/src/aws/sdk/kotlin/services/s3/{internal => express}/DefaultS3ExpressCredentialsProvider.kt (91%) rename services/s3/common/src/aws/sdk/kotlin/services/s3/{ => express}/S3ExpressCrc32ChecksumInterceptor.kt (98%) rename services/s3/common/src/aws/sdk/kotlin/services/s3/{internal => express}/S3ExpressCredentialsCache.kt (99%) rename services/s3/common/src/aws/sdk/kotlin/services/s3/{ => express}/S3ExpressCredentialsProvider.kt (90%) rename services/s3/common/src/aws/sdk/kotlin/services/s3/{ => express}/S3ExpressDisableChecksumInterceptor.kt (97%) rename services/s3/common/src/aws/sdk/kotlin/services/s3/{ => express}/S3ExpressHttpSigner.kt (97%) rename services/s3/common/test/aws/sdk/kotlin/services/s3/{internal => express}/S3ExpressCredentialsCacheTest.kt (94%) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index 6a20dae0edc..f9b36fc3098 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -76,7 +76,7 @@ class ClientConfigIntegration : KotlinIntegration { symbol = buildSymbol { name = "S3ExpressCredentialsProvider" nullable = false - namespace = "aws.sdk.kotlin.services.s3" + namespace = "aws.sdk.kotlin.services.s3.express" } documentation = """ Credentials provider to be used for making requests to S3 Express. diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt index cc2d1ffbda4..33c78d74795 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt @@ -2,9 +2,9 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ - package aws.sdk.kotlin.services.s3 +import aws.sdk.kotlin.services.s3.express.S3ExpressHttpSigner import aws.smithy.kotlin.runtime.InternalApi import aws.smithy.kotlin.runtime.auth.AuthOption import aws.smithy.kotlin.runtime.auth.AuthSchemeId @@ -15,6 +15,7 @@ import aws.smithy.kotlin.runtime.http.auth.AwsHttpSigner import aws.smithy.kotlin.runtime.http.auth.HttpSigner import aws.smithy.kotlin.runtime.http.auth.sigV4 +// FIXME Had trouble moving this to `express` package because this val became unresolved symbol public val AuthSchemeId.Companion.AwsSigV4S3Express: AuthSchemeId get() = AuthSchemeId("aws.auth#sigv4s3express") diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt similarity index 91% rename from services/s3/common/src/aws/sdk/kotlin/services/s3/internal/DefaultS3ExpressCredentialsProvider.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index a0ec134e313..17e346b70d8 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -2,11 +2,10 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.services.s3.internal +package aws.sdk.kotlin.services.s3.express import aws.sdk.kotlin.services.s3.S3Attributes import aws.sdk.kotlin.services.s3.S3Client -import aws.sdk.kotlin.services.s3.S3ExpressCredentialsProvider import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider import aws.smithy.kotlin.runtime.collections.Attributes diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCrc32ChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt similarity index 98% rename from services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCrc32ChecksumInterceptor.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt index aa8ad91fc5d..6172bb7348d 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCrc32ChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt @@ -2,7 +2,7 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.services.s3 +package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext import aws.smithy.kotlin.runtime.collections.AttributeKey diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt similarity index 99% rename from services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index 410deb3103d..e6664d15612 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -2,7 +2,7 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.services.s3.internal +package aws.sdk.kotlin.services.s3.express import aws.sdk.kotlin.services.s3.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt similarity index 90% rename from services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt index 2cdbc139305..89e9c99a196 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt @@ -2,9 +2,8 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.services.s3 +package aws.sdk.kotlin.services.s3.express -import aws.sdk.kotlin.services.s3.internal.DefaultS3ExpressCredentialsProvider import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvider import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt similarity index 97% rename from services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressDisableChecksumInterceptor.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt index 536b14ca70f..60c3318757b 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressDisableChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt @@ -2,7 +2,7 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.services.s3 +package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext import aws.smithy.kotlin.runtime.collections.AttributeKey diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressHttpSigner.kt similarity index 97% rename from services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressHttpSigner.kt index 69abcdc0f46..29224098c4b 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3ExpressHttpSigner.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressHttpSigner.kt @@ -2,7 +2,7 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.services.s3 +package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningAttributes diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt similarity index 94% rename from services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt rename to services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt index 615f793b5ef..64b6084e490 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/internal/S3ExpressCredentialsCacheTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt @@ -2,7 +2,7 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.services.s3.internal +package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import kotlinx.coroutines.test.runTest From 2cbf22b856970714723feb9cfe32f5a0963a52af Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 15:45:16 -0500 Subject: [PATCH 086/145] Remove FIXME comment --- services/s3/e2eTest/src/S3TestUtils.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 06cf750660b..4b61a1e5bc6 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -64,9 +64,6 @@ object S3TestUtils { } } - // FIXME skipping putBucketLifecycleConfiguration because the following exception is thrown from S3 if I try to do it: - // aws.sdk.kotlin.services.s3.model.S3Exception: LifecycleConfiguration is not valid, expected CreateBucketConfiguration - testBucket } From 8ec0916a41395b728d75890f6a6f6cd110ffeacc Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 15:45:36 -0500 Subject: [PATCH 087/145] Fix logging during async refresh --- .../kotlin/services/s3/express/S3ExpressCredentialsCache.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index e6664d15612..5a2cd21e547 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -8,6 +8,7 @@ import aws.sdk.kotlin.services.s3.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.collections.LruCache import aws.smithy.kotlin.runtime.io.Closeable +import aws.smithy.kotlin.runtime.telemetry.TelemetryProviderContext import aws.smithy.kotlin.runtime.telemetry.logging.logger import aws.smithy.kotlin.runtime.time.Clock import aws.smithy.kotlin.runtime.time.until @@ -25,7 +26,7 @@ internal class S3ExpressCredentialsCache( private val client: S3Client, private val clock: Clock = Clock.System, ) : CoroutineScope, Closeable { - override val coroutineContext: CoroutineContext = Job() + CoroutineName("S3ExpressCredentialsCacheRefresh") + override val coroutineContext: CoroutineContext = Job() + CoroutineName("S3ExpressCredentialsCacheRefresh") + TelemetryProviderContext(client.config.telemetryProvider) private val lru = LruCache(DEFAULT_S3_EXPRESS_CACHE_SIZE) From 99f63c00f44efc5b3dd9a36d2d42b0fc4ef8b826 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 18:27:18 -0500 Subject: [PATCH 088/145] Refactor to use `TimeSource` --- .../s3/express/S3ExpressCredentialsCache.kt | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index 5a2cd21e547..4323151d0aa 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -12,18 +12,21 @@ import aws.smithy.kotlin.runtime.telemetry.TelemetryProviderContext import aws.smithy.kotlin.runtime.telemetry.logging.logger import aws.smithy.kotlin.runtime.time.Clock import aws.smithy.kotlin.runtime.time.until -import aws.smithy.kotlin.runtime.util.ExpiringValue import kotlinx.coroutines.* import kotlin.coroutines.CoroutineContext +import kotlin.time.Duration import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeMark +import kotlin.time.TimeSource private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 private val REFRESH_BUFFER = 1.minutes -private val DEFAULT_REFRESH_PERIOD = 5.minutes +private val DEFAULT_REFRESH_PERIOD = 3.minutes internal class S3ExpressCredentialsCache( private val client: S3Client, + private val timeSource: TimeSource = TimeSource.Monotonic, private val clock: Clock = Clock.System, ) : CoroutineScope, Closeable { override val coroutineContext: CoroutineContext = Job() + CoroutineName("S3ExpressCredentialsCacheRefresh") + TelemetryProviderContext(client.config.telemetryProvider) @@ -38,7 +41,7 @@ internal class S3ExpressCredentialsCache( suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru .get(key) - ?.takeIf { !it.expiringCredentials.isExpired(clock) } + ?.takeIf { !it.expiringCredentials.isExpired } ?.let { it.usedSinceLastRefresh = true it.expiringCredentials.value @@ -49,7 +52,7 @@ internal class S3ExpressCredentialsCache( lru.put(key, S3ExpressCredentialsCacheValue(value, usedSinceLastRefresh = true)) } - private fun ExpiringValue.isExpired(clock: Clock): Boolean = clock.now().until(expiresAt).absoluteValue <= REFRESH_BUFFER + private val ExpiringValue.isExpired: Boolean get() = (expiresAt + REFRESH_BUFFER).hasPassedNow() /** * Attempt to refresh the credentials in the cache. A refresh is initiated when the `nextRefresh` time has been reached, @@ -78,26 +81,31 @@ internal class S3ExpressCredentialsCache( } // Refresh any credentials which are already expired - val expiredEntries = entries.filter { it.value.expiringCredentials.isExpired(clock) } + val expiredEntries = entries.filter { it.value.expiringCredentials.isExpired } expiredEntries.forEach { entry -> logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..." } lru.put(entry.key, S3ExpressCredentialsCacheValue(createSessionCredentials(entry.key.bucket), false)) } // Find the next expiring credentials, sleep until then - val nextExpiringEntry = entries.minByOrNull { it.value.expiringCredentials.expiresAt } + val nextExpiringEntry = entries.maxByOrNull { + // note: `expiresAt` is always in the future, so the `elapsedNow` values are negative. + // that's the reason `maxBy` is used instead of `minBy` + it.value.expiringCredentials.expiresAt.elapsedNow() + } - val nextRefresh = nextExpiringEntry?.let { - minOf(clock.now() + DEFAULT_REFRESH_PERIOD, it.value.expiringCredentials.expiresAt - REFRESH_BUFFER) - } ?: (clock.now() + DEFAULT_REFRESH_PERIOD) + val delayDuration = nextExpiringEntry + ?.let { timeSource.markNow().until(it.value.expiringCredentials.expiresAt) } + ?: DEFAULT_REFRESH_PERIOD - logger.debug { "Completed credentials refresh, next attempt in ${clock.now().until(nextRefresh)}" } - delay(clock.now().until(nextRefresh)) + logger.debug { "Completed credentials refresh, next attempt in $delayDuration" } + delay(delayDuration) } } private suspend fun createSessionCredentials(bucket: String): ExpiringValue { val credentials = client.createSession { this.bucket = bucket }.credentials!! + val expirationTimeMark = timeSource.markNow() + clock.now().until(credentials.expiration) return ExpiringValue( Credentials( @@ -107,7 +115,7 @@ internal class S3ExpressCredentialsCache( expiration = credentials.expiration, providerName = "DefaultS3ExpressCredentialsProvider", ), - credentials.expiration, + expirationTimeMark, ) } @@ -125,3 +133,13 @@ internal data class S3ExpressCredentialsCacheValue( val expiringCredentials: ExpiringValue, var usedSinceLastRefresh: Boolean, ) + +/** + * Get the [Duration] between [this] TimeMark and an [other] TimeMark + */ +internal fun TimeMark.until(other: TimeMark): Duration = (this.elapsedNow().absoluteValue - other.elapsedNow().absoluteValue).absoluteValue + +/** + * A value with an expiration [TimeMark] + */ +internal data class ExpiringValue (val value: T, val expiresAt: TimeMark) From de2ea56f20c530a1fd34b4329d357b568070368c Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Sat, 17 Feb 2024 18:34:03 -0500 Subject: [PATCH 089/145] `supervisorJob` --- .../s3/express/S3ExpressCredentialsCache.kt | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index 4323151d0aa..c3d42dfe128 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -82,14 +82,24 @@ internal class S3ExpressCredentialsCache( // Refresh any credentials which are already expired val expiredEntries = entries.filter { it.value.expiringCredentials.isExpired } - expiredEntries.forEach { entry -> - logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..." } - lru.put(entry.key, S3ExpressCredentialsCacheValue(createSessionCredentials(entry.key.bucket), false)) + + supervisorScope { + expiredEntries.forEach { entry -> + logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..." } + + try { + val refreshed = async { createSessionCredentials(entry.key.bucket) }.await() + lru.put(entry.key, S3ExpressCredentialsCacheValue(refreshed, false)) + } catch (e: Exception) { + logger.warn(e) { "Failed to refresh credentials for ${entry.key.bucket}" } + } + + } } // Find the next expiring credentials, sleep until then val nextExpiringEntry = entries.maxByOrNull { - // note: `expiresAt` is always in the future, so the `elapsedNow` values are negative. + // note: `expiresAt` is always in the future, which means the `elapsedNow` values are negative. // that's the reason `maxBy` is used instead of `minBy` it.value.expiringCredentials.expiresAt.elapsedNow() } From 416fa6c88a9d256b67d382e5286bf5be3d314e01 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Mon, 19 Feb 2024 12:14:14 -0500 Subject: [PATCH 090/145] Apply synthetic trait --- .../s3/express/S3ExpressIntegration.kt | 47 +++++++++---------- .../s3/express/SigV4S3ExpressAuthTrait.kt | 14 ++++++ 2 files changed, 36 insertions(+), 25 deletions(-) create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthTrait.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index f5fdb9e2567..a03eb1c26cb 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -4,21 +4,16 @@ */ package aws.sdk.kotlin.codegen.customization.s3.express -import aws.sdk.kotlin.codegen.customization.s3.ClientConfigIntegration +import SigV4S3ExpressAuthTrait import aws.sdk.kotlin.codegen.customization.s3.isS3 import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.* -import software.amazon.smithy.kotlin.codegen.integration.AppendingSectionWriter import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration -import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding import software.amazon.smithy.kotlin.codegen.model.buildSymbol import software.amazon.smithy.kotlin.codegen.model.expectShape import software.amazon.smithy.kotlin.codegen.model.getTrait import software.amazon.smithy.kotlin.codegen.model.hasTrait -import software.amazon.smithy.kotlin.codegen.rendering.auth.AuthSchemeProviderGenerator -import software.amazon.smithy.kotlin.codegen.rendering.auth.IdentityProviderConfigGenerator -import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpProtocolClientGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware import software.amazon.smithy.kotlin.codegen.utils.dq @@ -27,8 +22,10 @@ import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape import software.amazon.smithy.model.shapes.ServiceShape import software.amazon.smithy.model.shapes.StructureShape +import software.amazon.smithy.model.traits.AuthTrait import software.amazon.smithy.model.traits.HttpChecksumRequiredTrait import software.amazon.smithy.model.traits.HttpHeaderTrait +import software.amazon.smithy.model.transform.ModelTransformer /** * An integration which handles codegen for S3 Express, such as: @@ -40,25 +37,25 @@ import software.amazon.smithy.model.traits.HttpHeaderTrait class S3ExpressIntegration : KotlinIntegration { override fun enabledForService(model: Model, settings: KotlinSettings) = model.expectShape(settings.service).isS3 - override val sectionWriters: List - get() = listOf( - SectionWriterBinding(HttpProtocolClientGenerator.ConfigureAuthSchemes, configureS3ExpressAuthSchemeWriter), - SectionWriterBinding(AuthSchemeProviderGenerator.ServiceDefaults, setServiceDefaultAuthOptionWriter), - SectionWriterBinding(IdentityProviderConfigGenerator.ConfigureIdentityProviderForAuthScheme, configureIdentityProviderForAuthSchemeWriter), - ) + /** + * Add a synthetic SigV4 S3 Express auth trait + */ + override fun preprocessModel(model: Model, settings: KotlinSettings): Model { + return ModelTransformer.create().mapShapes(model) { shape -> + when { + shape.isServiceShape -> { + val builder = (shape as ServiceShape).toBuilder() - private val configureS3ExpressAuthSchemeWriter = AppendingSectionWriter { writer -> - writer.withBlock("getOrPut(#T) {", "}", SigV4S3ExpressAuthSchemeHandler().authSchemeIdSymbol) { - writer.write("#T(#T, #S)", SigV4S3ExpressAuthSchemeSymbol, RuntimeTypes.Auth.Signing.AwsSigningStandard.DefaultAwsSigner, "s3") - } - } + builder.addTrait(SigV4S3ExpressAuthTrait()) - private val setServiceDefaultAuthOptionWriter = AppendingSectionWriter { writer -> - writer.write("#T(),", sigV4S3ExpressSymbol) - } + val authTrait = AuthTrait(mutableSetOf(SigV4S3ExpressAuthTrait.ID) + shape.expectTrait(AuthTrait::class.java).valueSet) + builder.addTrait(authTrait) - private val configureIdentityProviderForAuthSchemeWriter = AppendingSectionWriter { writer -> - writer.write("#S -> config.#L", "aws.auth#sigv4s3express", ClientConfigIntegration.S3ExpressCredentialsProvider.propertyName) + builder.build() + } + else -> shape + } + } } override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = @@ -96,7 +93,7 @@ class S3ExpressIntegration : KotlinIntegration { name = "S3Attributes" namespace = "aws.sdk.kotlin.services.s3" } - writer.write("input.bucket?.let { op.context[#T.DirectoryBucket] = it }", attributesSymbol) + writer.write("input.bucket?.let { op.context[#T.Bucket] = it }", attributesSymbol) } } @@ -113,7 +110,7 @@ class S3ExpressIntegration : KotlinIntegration { override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { val interceptorSymbol = buildSymbol { - namespace = "aws.sdk.kotlin.services.s3" + namespace = "aws.sdk.kotlin.services.s3.express" name = "S3ExpressCrc32ChecksumInterceptor" } @@ -141,7 +138,7 @@ class S3ExpressIntegration : KotlinIntegration { override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { val interceptorSymbol = buildSymbol { - namespace = "aws.sdk.kotlin.services.s3" + namespace = "aws.sdk.kotlin.services.s3.express" name = "S3ExpressDisableChecksumInterceptor" } writer.addImport(interceptorSymbol) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthTrait.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthTrait.kt new file mode 100644 index 00000000000..e572f00949c --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthTrait.kt @@ -0,0 +1,14 @@ +import software.amazon.smithy.model.node.Node +import software.amazon.smithy.model.shapes.ShapeId +import software.amazon.smithy.model.traits.AbstractTrait + +/** + * Synthetic auth trait applied to S3's model to enable SigV4 S3 Express auth scheme. + */ +internal class SigV4S3ExpressAuthTrait : AbstractTrait(ID, Node.objectNode()) { + companion object { + val ID = ShapeId.from("aws.auth#sigv4s3express") + } + override fun createNode(): Node = Node.objectNode() + override fun isSynthetic(): Boolean = true +} \ No newline at end of file From 7e2cffc656ea3d432e103e23cbec638f8b406d8b Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Mon, 19 Feb 2024 17:23:17 -0500 Subject: [PATCH 091/145] Refactor async refresh logic to DefaultS3ExpressCredentialsProvider --- .../s3/ClientConfigIntegration.kt | 2 +- .../DefaultS3ExpressCredentialsProvider.kt | 133 ++++++++++++++++-- .../s3/express/S3ExpressCredentialsCache.kt | 126 +++-------------- .../express/S3ExpressCredentialsProvider.kt | 8 +- 4 files changed, 144 insertions(+), 125 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index f9b36fc3098..d5a99cd9c54 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -84,7 +84,7 @@ class ClientConfigIntegration : KotlinIntegration { propertyType = ConfigPropertyType.Custom( render = { _, writer -> - writer.write("public val #1L: #2T = builder.#1L ?: #2T.default(this.credentialsProvider)", name, symbol) + writer.write("public val #1L: #2T = builder.#1L ?: #2T.default()", name, symbol) }, renderBuilder = { prop, writer -> prop.documentation?.let(writer::dokka) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 17e346b70d8..87941152f90 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -6,27 +6,138 @@ package aws.sdk.kotlin.services.s3.express import aws.sdk.kotlin.services.s3.S3Attributes import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.* +import aws.smithy.kotlin.runtime.ExperimentalApi import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.io.SdkManagedBase +import aws.smithy.kotlin.runtime.telemetry.logging.Logger +import aws.smithy.kotlin.runtime.telemetry.logging.getLogger +import aws.smithy.kotlin.runtime.time.Clock +import aws.smithy.kotlin.runtime.time.until +import kotlinx.coroutines.* +import kotlin.coroutines.CoroutineContext +import kotlin.time.Duration +import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds +import kotlin.time.TimeMark +import kotlin.time.TimeSource + +internal val REFRESH_BUFFER = 1.minutes +private val DEFAULT_REFRESH_PERIOD = 3.minutes +private const val CREDENTIALS_PROVIDER_NAME = "DefaultS3ExpressCredentialsProvider" internal class DefaultS3ExpressCredentialsProvider( - val bootstrapCredentialsProvider: CredentialsProvider, -) : S3ExpressCredentialsProvider, SdkManagedBase() { - private lateinit var credentialsCache: S3ExpressCredentialsCache + private val timeSource: TimeSource = TimeSource.Monotonic, + private val clock: Clock = Clock.System, +) : S3ExpressCredentialsProvider, SdkManagedBase(), CoroutineScope { + private lateinit var client: S3Client + private lateinit var logger: Logger + private val credentialsCache = S3ExpressCredentialsCache() - override suspend fun resolve(attributes: Attributes): Credentials { - if (!this::credentialsCache.isInitialized) { - credentialsCache = S3ExpressCredentialsCache(attributes[S3Attributes.ExpressClient] as S3Client) + override val coroutineContext: CoroutineContext = Job() + + CoroutineName(CREDENTIALS_PROVIDER_NAME) + + init { + launch(coroutineContext) { + refresh() } + } + + @OptIn(ExperimentalApi::class) + override suspend fun resolve(attributes: Attributes): Credentials { + client = (attributes[S3Attributes.ExpressClient] as S3Client) + logger = client.config.telemetryProvider.loggerProvider.getLogger() + + val key = S3ExpressCredentialsCacheKey(attributes[S3Attributes.Bucket], client.config.credentialsProvider.resolve(attributes)) - val key = S3ExpressCredentialsCacheKey(attributes[S3Attributes.Bucket], bootstrapCredentialsProvider.resolve(attributes)) - return credentialsCache.get(key) + return credentialsCache.get(key)?.takeIf { !it.isExpired }?.value + ?: createSessionCredentials(key.bucket).also { credentialsCache.put(key, it) }.value } - override fun close() { - credentialsCache.close() + override fun close() = coroutineContext.cancel(null) + + + /** + * Attempt to refresh the credentials in the cache. A refresh is initiated when the `nextRefresh` time has been reached, + * which is either `DEFAULT_REFRESH_PERIOD` or the soonest credentials expiration time (minus a buffer), whichever comes first. + */ + private suspend fun refresh() { + while (isActive) { + if (!this::client.isInitialized) { + delay(5.seconds) + continue + } else if (credentialsCache.size == 0) { + logger.trace { "Cache is empty, waiting..." } + delay(5.seconds) + continue + } + + val entries = credentialsCache.entries + + // Evict any credentials that weren't used since the last refresh + entries.filter { !it.value.usedSinceLastRefresh }.forEach { + logger.debug { "Credentials for ${it.key.bucket} were not used since last refresh, evicting..." } + credentialsCache.remove(it.key) + } + + // Mark all credentials as not used since last refresh + entries.forEach { + it.value.usedSinceLastRefresh = false + } + + // Refresh any credentials which are already expired + val expiredEntries = entries.filter { it.value.expiringCredentials.isExpired } + + supervisorScope { + expiredEntries.forEach { entry -> + logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..." } + + try { + val refreshed = async { createSessionCredentials(entry.key.bucket) }.await() + credentialsCache.put(entry.key, refreshed, false) + } catch (e: Exception) { + logger.warn(e) { "Failed to refresh credentials for ${entry.key.bucket}" } + } + + } + } + + // Find the next expiring credentials, sleep until then + val nextExpiringEntry = entries.maxByOrNull { + // note: `expiresAt` is always in the future, which means the `elapsedNow` values are negative. + // that's the reason `maxBy` is used instead of `minBy` + it.value.expiringCredentials.expiresAt.elapsedNow() + } + + val delayDuration = nextExpiringEntry + ?.let { timeSource.markNow().until(it.value.expiringCredentials.expiresAt) } + ?: DEFAULT_REFRESH_PERIOD + + logger.debug { "Completed credentials refresh, next attempt in $delayDuration" } + delay(delayDuration) + } + } + + private suspend fun createSessionCredentials(bucket: String): ExpiringValue { + val credentials = client.createSession { this.bucket = bucket }.credentials!! + val expirationTimeMark = timeSource.markNow() + clock.now().until(credentials.expiration) + + return ExpiringValue( + Credentials( + accessKeyId = credentials.accessKeyId, + secretAccessKey = credentials.secretAccessKey, + sessionToken = credentials.sessionToken, + expiration = credentials.expiration, + providerName = CREDENTIALS_PROVIDER_NAME , + ), + expirationTimeMark, + ) } } + +/** + * Get the [Duration] between [this] TimeMark and an [other] TimeMark + */ +internal fun TimeMark.until(other: TimeMark): Duration = (this.elapsedNow().absoluteValue - other.elapsedNow().absoluteValue).absoluteValue diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index c3d42dfe128..a215937f2f7 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -21,117 +21,34 @@ import kotlin.time.TimeMark import kotlin.time.TimeSource private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 -private val REFRESH_BUFFER = 1.minutes -private val DEFAULT_REFRESH_PERIOD = 3.minutes - -internal class S3ExpressCredentialsCache( - private val client: S3Client, - private val timeSource: TimeSource = TimeSource.Monotonic, - private val clock: Clock = Clock.System, -) : CoroutineScope, Closeable { - override val coroutineContext: CoroutineContext = Job() + CoroutineName("S3ExpressCredentialsCacheRefresh") + TelemetryProviderContext(client.config.telemetryProvider) +internal class S3ExpressCredentialsCache { private val lru = LruCache(DEFAULT_S3_EXPRESS_CACHE_SIZE) - init { - launch(coroutineContext) { - refresh() - } - } - - suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru - .get(key) - ?.takeIf { !it.expiringCredentials.isExpired } - ?.let { - it.usedSinceLastRefresh = true - it.expiringCredentials.value - } - ?: (createSessionCredentials(key.bucket).also { put(key, it) }).value - - suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue) { - lru.put(key, S3ExpressCredentialsCacheValue(value, usedSinceLastRefresh = true)) - } - - private val ExpiringValue.isExpired: Boolean get() = (expiresAt + REFRESH_BUFFER).hasPassedNow() - - /** - * Attempt to refresh the credentials in the cache. A refresh is initiated when the `nextRefresh` time has been reached, - * which is either `DEFAULT_REFRESH_PERIOD` or the soonest credentials expiration time (minus a buffer), whichever comes first. - */ - private suspend fun refresh() { - val logger = coroutineContext.logger() - while (isActive) { - if (lru.size == 0) { - logger.trace { "Cache is empty, waiting..." } - delay(5.seconds) - continue - } - - val entries = lru.entries - - // Evict any credentials that weren't used since the last refresh - entries.filter { !it.value.usedSinceLastRefresh }.forEach { - logger.debug { "Credentials for ${it.key.bucket} were not used since last refresh, evicting..." } - lru.remove(it.key) - } - - // Mark all credentials as not used since last refresh - entries.forEach { - it.value.usedSinceLastRefresh = false - } + suspend fun get(key: S3ExpressCredentialsCacheKey): ExpiringValue? = lru.get(key)?.expiringCredentials - // Refresh any credentials which are already expired - val expiredEntries = entries.filter { it.value.expiringCredentials.isExpired } + suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue, usedSinceLastRefresh: Boolean = true) = + lru.put(key, S3ExpressCredentialsCacheValue(value, usedSinceLastRefresh)) - supervisorScope { - expiredEntries.forEach { entry -> - logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..." } + suspend fun remove(key: S3ExpressCredentialsCacheKey) : ExpiringValue? = + lru.remove(key)?.expiringCredentials - try { - val refreshed = async { createSessionCredentials(entry.key.bucket) }.await() - lru.put(entry.key, S3ExpressCredentialsCacheValue(refreshed, false)) - } catch (e: Exception) { - logger.warn(e) { "Failed to refresh credentials for ${entry.key.bucket}" } - } + public val size: Int + get() = lru.size - } - } + public val entries: Set> + get() = lru.entries - // Find the next expiring credentials, sleep until then - val nextExpiringEntry = entries.maxByOrNull { - // note: `expiresAt` is always in the future, which means the `elapsedNow` values are negative. - // that's the reason `maxBy` is used instead of `minBy` - it.value.expiringCredentials.expiresAt.elapsedNow() - } +// suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru +// .get(key) +// ?.takeIf { !it.expiringCredentials.isExpired } +// ?.let { +// it.usedSinceLastRefresh = true +// it.expiringCredentials.value +// } +// ?: (createSessionCredentials(key.bucket).also { put(key, it) }).value - val delayDuration = nextExpiringEntry - ?.let { timeSource.markNow().until(it.value.expiringCredentials.expiresAt) } - ?: DEFAULT_REFRESH_PERIOD - logger.debug { "Completed credentials refresh, next attempt in $delayDuration" } - delay(delayDuration) - } - } - - private suspend fun createSessionCredentials(bucket: String): ExpiringValue { - val credentials = client.createSession { this.bucket = bucket }.credentials!! - val expirationTimeMark = timeSource.markNow() + clock.now().until(credentials.expiration) - - return ExpiringValue( - Credentials( - accessKeyId = credentials.accessKeyId, - secretAccessKey = credentials.secretAccessKey, - sessionToken = credentials.sessionToken, - expiration = credentials.expiration, - providerName = "DefaultS3ExpressCredentialsProvider", - ), - expirationTimeMark, - ) - } - - override fun close() { - coroutineContext.cancel(null) - } } internal data class S3ExpressCredentialsCacheKey( @@ -144,12 +61,9 @@ internal data class S3ExpressCredentialsCacheValue( var usedSinceLastRefresh: Boolean, ) -/** - * Get the [Duration] between [this] TimeMark and an [other] TimeMark - */ -internal fun TimeMark.until(other: TimeMark): Duration = (this.elapsedNow().absoluteValue - other.elapsedNow().absoluteValue).absoluteValue - /** * A value with an expiration [TimeMark] */ internal data class ExpiringValue (val value: T, val expiresAt: TimeMark) + +internal val ExpiringValue.isExpired: Boolean get() = (expiresAt + REFRESH_BUFFER).hasPassedNow() \ No newline at end of file diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt index 89e9c99a196..df54698e0fd 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt @@ -5,9 +5,6 @@ package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvider -import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials -import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider -import aws.smithy.kotlin.runtime.collections.Attributes /** * A credentials provider used for making requests to S3 Express One Zone directory buckets. @@ -16,10 +13,7 @@ public interface S3ExpressCredentialsProvider : CloseableCredentialsProvider { public companion object { /** * Create an instance of the default [S3ExpressCredentialsProvider] implementation - * @param bootstrapCredentialsProvider the credentials provider used to call s3:CreateSession to retrieve S3 Express - * credentials */ - public fun default(bootstrapCredentialsProvider: CredentialsProvider): S3ExpressCredentialsProvider = - DefaultS3ExpressCredentialsProvider(bootstrapCredentialsProvider) + public fun default(): S3ExpressCredentialsProvider = DefaultS3ExpressCredentialsProvider() } } From ec1dabb5886867173320c0c2983a7d482cf4ed91 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 20 Feb 2024 09:48:48 -0500 Subject: [PATCH 092/145] simplify cache, perform refresh in provider --- .../DefaultS3ExpressCredentialsProvider.kt | 43 +++++++++++++------ .../s3/express/S3ExpressCredentialsCache.kt | 21 ++------- .../express/S3ExpressCredentialsCacheTest.kt | 40 ++++++++++++++++- 3 files changed, 72 insertions(+), 32 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 87941152f90..6841543f325 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -12,7 +12,7 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.io.SdkManagedBase -import aws.smithy.kotlin.runtime.telemetry.logging.Logger +import aws.smithy.kotlin.runtime.telemetry.TelemetryProvider import aws.smithy.kotlin.runtime.telemetry.logging.getLogger import aws.smithy.kotlin.runtime.time.Clock import aws.smithy.kotlin.runtime.time.until @@ -24,20 +24,32 @@ import kotlin.time.Duration.Companion.seconds import kotlin.time.TimeMark import kotlin.time.TimeSource +/** + * The duration before expiration that [Credentials] are considered expired + */ internal val REFRESH_BUFFER = 1.minutes + +/** + * How long to wait between cache refresh attempts if no [Credentials] are in the cache + */ private val DEFAULT_REFRESH_PERIOD = 3.minutes + private const val CREDENTIALS_PROVIDER_NAME = "DefaultS3ExpressCredentialsProvider" +/** + * The default implementation of [S3ExpressCredentialsProvider] + * @param timeSource the time source to use. defaults to [TimeSource.Monotonic] + * @param clock the clock to use. defaults to [Clock.System]. note: the clock is only used to get an initial [Duration] + * until credentials expiration. + */ internal class DefaultS3ExpressCredentialsProvider( private val timeSource: TimeSource = TimeSource.Monotonic, private val clock: Clock = Clock.System, ) : S3ExpressCredentialsProvider, SdkManagedBase(), CoroutineScope { private lateinit var client: S3Client - private lateinit var logger: Logger private val credentialsCache = S3ExpressCredentialsCache() - override val coroutineContext: CoroutineContext = Job() + - CoroutineName(CREDENTIALS_PROVIDER_NAME) + override val coroutineContext: CoroutineContext = Job() + CoroutineName(CREDENTIALS_PROVIDER_NAME) init { launch(coroutineContext) { @@ -45,20 +57,17 @@ internal class DefaultS3ExpressCredentialsProvider( } } - @OptIn(ExperimentalApi::class) override suspend fun resolve(attributes: Attributes): Credentials { - client = (attributes[S3Attributes.ExpressClient] as S3Client) - logger = client.config.telemetryProvider.loggerProvider.getLogger() + client = attributes[S3Attributes.ExpressClient] as S3Client val key = S3ExpressCredentialsCacheKey(attributes[S3Attributes.Bucket], client.config.credentialsProvider.resolve(attributes)) - return credentialsCache.get(key)?.takeIf { !it.isExpired }?.value - ?: createSessionCredentials(key.bucket).also { credentialsCache.put(key, it) }.value + return credentialsCache.get(key)?.expiringCredentials?.takeIf { !it.isExpired }?.value + ?: createSessionCredentials(key.bucket).also { credentialsCache.put(key, S3ExpressCredentialsCacheValue(it, usedSinceLastRefresh = true)) }.value } override fun close() = coroutineContext.cancel(null) - /** * Attempt to refresh the credentials in the cache. A refresh is initiated when the `nextRefresh` time has been reached, * which is either `DEFAULT_REFRESH_PERIOD` or the soonest credentials expiration time (minus a buffer), whichever comes first. @@ -96,7 +105,7 @@ internal class DefaultS3ExpressCredentialsProvider( try { val refreshed = async { createSessionCredentials(entry.key.bucket) }.await() - credentialsCache.put(entry.key, refreshed, false) + credentialsCache.put(entry.key, S3ExpressCredentialsCacheValue(refreshed, usedSinceLastRefresh = false)) } catch (e: Exception) { logger.warn(e) { "Failed to refresh credentials for ${entry.key.bucket}" } } @@ -106,8 +115,8 @@ internal class DefaultS3ExpressCredentialsProvider( // Find the next expiring credentials, sleep until then val nextExpiringEntry = entries.maxByOrNull { - // note: `expiresAt` is always in the future, which means the `elapsedNow` values are negative. - // that's the reason `maxBy` is used instead of `minBy` + // note: `expiresAt` is a future time, which means the `elapsedNow` values are negative + // and count up until expiration at t=0. that's why `maxBy` is used instead of `minBy` it.value.expiringCredentials.expiresAt.elapsedNow() } @@ -135,8 +144,16 @@ internal class DefaultS3ExpressCredentialsProvider( expirationTimeMark, ) } + + @OptIn(ExperimentalApi::class) + internal val logger get() = if (this::client.isInitialized) { + client.config.telemetryProvider.loggerProvider.getLogger() + } else { + TelemetryProvider.None.loggerProvider.getLogger() + } } + /** * Get the [Duration] between [this] TimeMark and an [other] TimeMark */ diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index a215937f2f7..fac9e99c504 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -25,30 +25,17 @@ private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 internal class S3ExpressCredentialsCache { private val lru = LruCache(DEFAULT_S3_EXPRESS_CACHE_SIZE) - suspend fun get(key: S3ExpressCredentialsCacheKey): ExpiringValue? = lru.get(key)?.expiringCredentials + suspend fun get(key: S3ExpressCredentialsCacheKey): S3ExpressCredentialsCacheValue? = lru.get(key) - suspend fun put(key: S3ExpressCredentialsCacheKey, value: ExpiringValue, usedSinceLastRefresh: Boolean = true) = - lru.put(key, S3ExpressCredentialsCacheValue(value, usedSinceLastRefresh)) + suspend fun put(key: S3ExpressCredentialsCacheKey, value: S3ExpressCredentialsCacheValue) = lru.put(key, value) - suspend fun remove(key: S3ExpressCredentialsCacheKey) : ExpiringValue? = - lru.remove(key)?.expiringCredentials + suspend fun remove(key: S3ExpressCredentialsCacheKey) : S3ExpressCredentialsCacheValue? = lru.remove(key) public val size: Int get() = lru.size public val entries: Set> get() = lru.entries - -// suspend fun get(key: S3ExpressCredentialsCacheKey): Credentials = lru -// .get(key) -// ?.takeIf { !it.expiringCredentials.isExpired } -// ?.let { -// it.usedSinceLastRefresh = true -// it.expiringCredentials.value -// } -// ?: (createSessionCredentials(key.bucket).also { put(key, it) }).value - - } internal data class S3ExpressCredentialsCacheKey( @@ -58,7 +45,7 @@ internal data class S3ExpressCredentialsCacheKey( internal data class S3ExpressCredentialsCacheValue( val expiringCredentials: ExpiringValue, - var usedSinceLastRefresh: Boolean, + var usedSinceLastRefresh: Boolean = false, ) /** diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt index 64b6084e490..2172a46a865 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt @@ -6,8 +6,9 @@ package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import kotlinx.coroutines.test.runTest -import kotlin.test.Test -import kotlin.test.assertEquals +import kotlin.test.* +import kotlin.time.TestTimeSource +import kotlin.time.Duration.Companion.seconds public class S3ExpressCredentialsCacheTest { @Test @@ -21,4 +22,39 @@ public class S3ExpressCredentialsCacheTest { assertEquals(key1, key2) } + + @Test + fun testCacheOperations() = runTest { + val cache = S3ExpressCredentialsCache() + + val bucket = "bucket" + val bootstrapCredentials = Credentials("accessKeyId", "secretAccessKey", "sessionToken") + val key = S3ExpressCredentialsCacheKey(bucket, bootstrapCredentials) + + val sessionCredentials = Credentials("superFastAccessKey", "superSecretSecretKey", "s3SessionToken") + val expiringSessionCredentials = ExpiringValue(sessionCredentials, TestTimeSource().markNow()) + val value = S3ExpressCredentialsCacheValue(expiringSessionCredentials) + + cache.put(key, value) // put + assertEquals(expiringSessionCredentials, cache.get(key)?.expiringCredentials) // get + assertEquals(1, cache.size) // size + assertContains(cache.entries.map { it.key }, key) // entries + assertContains(cache.entries.map { it.value }, value) // entries + + cache.remove(key) + assertEquals(0, cache.size) + assertNull(cache.get(key)) + } + + @Test + fun testIsExpired() = runTest { + val timeSource = TestTimeSource() + + val sessionCredentials = Credentials("superFastAccessKey", "superSecretSecretKey", "s3SessionToken") + val expiringSessionCredentials = ExpiringValue(sessionCredentials, timeSource.markNow()) + + assertFalse(expiringSessionCredentials.isExpired) + timeSource += 1.seconds + assertTrue(expiringSessionCredentials.isExpired) + } } From fdfc4753476d7e8414ac3ed7e7fa4e67662a0e11 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 20 Feb 2024 09:50:48 -0500 Subject: [PATCH 093/145] ktlint --- .../s3/express/S3ExpressIntegration.kt | 21 ++++++++++++------- .../s3/express/SigV4S3ExpressAuthTrait.kt | 6 +++++- .../DefaultS3ExpressCredentialsProvider.kt | 6 ++---- .../s3/express/S3ExpressCredentialsCache.kt | 16 ++------------ .../express/S3ExpressCredentialsCacheTest.kt | 2 +- 5 files changed, 23 insertions(+), 28 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index a03eb1c26cb..80812e86845 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -35,28 +35,30 @@ import software.amazon.smithy.model.transform.ModelTransformer * 4. Disable all checksums for s3:UploadPart */ class S3ExpressIntegration : KotlinIntegration { - override fun enabledForService(model: Model, settings: KotlinSettings) = model.expectShape(settings.service).isS3 + override fun enabledForService(model: Model, settings: KotlinSettings) = + model.expectShape(settings.service).isS3 /** * Add a synthetic SigV4 S3 Express auth trait */ - override fun preprocessModel(model: Model, settings: KotlinSettings): Model { - return ModelTransformer.create().mapShapes(model) { shape -> + override fun preprocessModel(model: Model, settings: KotlinSettings): Model = + ModelTransformer.create().mapShapes(model) { shape -> when { shape.isServiceShape -> { val builder = (shape as ServiceShape).toBuilder() builder.addTrait(SigV4S3ExpressAuthTrait()) - val authTrait = AuthTrait(mutableSetOf(SigV4S3ExpressAuthTrait.ID) + shape.expectTrait(AuthTrait::class.java).valueSet) + val authTrait = + AuthTrait(mutableSetOf(SigV4S3ExpressAuthTrait.ID) + shape.expectTrait(AuthTrait::class.java).valueSet) builder.addTrait(authTrait) builder.build() } + else -> shape } } - } override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = resolved + listOf( @@ -69,7 +71,8 @@ class S3ExpressIntegration : KotlinIntegration { private val AddClientToExecutionContext = object : ProtocolMiddleware { override val name: String = "AddClientToExecutionContext" - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = ctx.model.expectShape(ctx.settings.service).isS3 + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + ctx.model.expectShape(ctx.settings.service).isS3 override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { val attributesSymbol = buildSymbol { @@ -123,7 +126,8 @@ class S3ExpressIntegration : KotlinIntegration { // S3 models a header name x-amz-sdk-checksum-algorithm representing the name of the checksum algorithm used val checksumHeaderName = checksumAlgorithmMember?.getTrait()?.value - val checksumRequired = op.hasTrait() || httpChecksumTrait?.isRequestChecksumRequired == true + val checksumRequired = + op.hasTrait() || httpChecksumTrait?.isRequestChecksumRequired == true if (checksumRequired) { writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq() ?: ""}))", interceptorSymbol) @@ -134,7 +138,8 @@ class S3ExpressIntegration : KotlinIntegration { private val UploadPartDisableChecksum = object : ProtocolMiddleware { override val name: String = "UploadPartDisableChecksum" - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = op.isS3UploadPart + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = + op.isS3UploadPart override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { val interceptorSymbol = buildSymbol { diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthTrait.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthTrait.kt index e572f00949c..f6f9ba29faf 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthTrait.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthTrait.kt @@ -1,3 +1,7 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ import software.amazon.smithy.model.node.Node import software.amazon.smithy.model.shapes.ShapeId import software.amazon.smithy.model.traits.AbstractTrait @@ -11,4 +15,4 @@ internal class SigV4S3ExpressAuthTrait : AbstractTrait(ID, Node.objectNode()) { } override fun createNode(): Node = Node.objectNode() override fun isSynthetic(): Boolean = true -} \ No newline at end of file +} diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 6841543f325..2887fefc0c9 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -4,9 +4,9 @@ */ package aws.sdk.kotlin.services.s3.express +import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.S3Attributes import aws.sdk.kotlin.services.s3.S3Client -import aws.sdk.kotlin.services.s3.* import aws.smithy.kotlin.runtime.ExperimentalApi import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.collections.Attributes @@ -109,7 +109,6 @@ internal class DefaultS3ExpressCredentialsProvider( } catch (e: Exception) { logger.warn(e) { "Failed to refresh credentials for ${entry.key.bucket}" } } - } } @@ -139,7 +138,7 @@ internal class DefaultS3ExpressCredentialsProvider( secretAccessKey = credentials.secretAccessKey, sessionToken = credentials.sessionToken, expiration = credentials.expiration, - providerName = CREDENTIALS_PROVIDER_NAME , + providerName = CREDENTIALS_PROVIDER_NAME, ), expirationTimeMark, ) @@ -153,7 +152,6 @@ internal class DefaultS3ExpressCredentialsProvider( } } - /** * Get the [Duration] between [this] TimeMark and an [other] TimeMark */ diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index fac9e99c504..6de3d6121d4 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -4,21 +4,9 @@ */ package aws.sdk.kotlin.services.s3.express -import aws.sdk.kotlin.services.s3.* import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.collections.LruCache -import aws.smithy.kotlin.runtime.io.Closeable -import aws.smithy.kotlin.runtime.telemetry.TelemetryProviderContext -import aws.smithy.kotlin.runtime.telemetry.logging.logger -import aws.smithy.kotlin.runtime.time.Clock -import aws.smithy.kotlin.runtime.time.until -import kotlinx.coroutines.* -import kotlin.coroutines.CoroutineContext -import kotlin.time.Duration -import kotlin.time.Duration.Companion.minutes -import kotlin.time.Duration.Companion.seconds import kotlin.time.TimeMark -import kotlin.time.TimeSource private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 @@ -29,7 +17,7 @@ internal class S3ExpressCredentialsCache { suspend fun put(key: S3ExpressCredentialsCacheKey, value: S3ExpressCredentialsCacheValue) = lru.put(key, value) - suspend fun remove(key: S3ExpressCredentialsCacheKey) : S3ExpressCredentialsCacheValue? = lru.remove(key) + suspend fun remove(key: S3ExpressCredentialsCacheKey): S3ExpressCredentialsCacheValue? = lru.remove(key) public val size: Int get() = lru.size @@ -53,4 +41,4 @@ internal data class S3ExpressCredentialsCacheValue( */ internal data class ExpiringValue (val value: T, val expiresAt: TimeMark) -internal val ExpiringValue.isExpired: Boolean get() = (expiresAt + REFRESH_BUFFER).hasPassedNow() \ No newline at end of file +internal val ExpiringValue.isExpired: Boolean get() = (expiresAt + REFRESH_BUFFER).hasPassedNow() diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt index 2172a46a865..dc802f6bf4b 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt @@ -7,8 +7,8 @@ package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import kotlinx.coroutines.test.runTest import kotlin.test.* -import kotlin.time.TestTimeSource import kotlin.time.Duration.Companion.seconds +import kotlin.time.TestTimeSource public class S3ExpressCredentialsCacheTest { @Test From ff98ba25007fd3241308e063920766f429cc790e Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 20 Feb 2024 09:58:40 -0500 Subject: [PATCH 094/145] Fix isExpired logic --- .../kotlin/services/s3/express/S3ExpressCredentialsCache.kt | 2 +- .../services/s3/express/S3ExpressCredentialsCacheTest.kt | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index 6de3d6121d4..51f6a7f9378 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -41,4 +41,4 @@ internal data class S3ExpressCredentialsCacheValue( */ internal data class ExpiringValue (val value: T, val expiresAt: TimeMark) -internal val ExpiringValue.isExpired: Boolean get() = (expiresAt + REFRESH_BUFFER).hasPassedNow() +internal val ExpiringValue.isExpired: Boolean get() = (expiresAt - REFRESH_BUFFER).hasPassedNow() diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt index dc802f6bf4b..2ae7da8b203 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt @@ -7,6 +7,7 @@ package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import kotlinx.coroutines.test.runTest import kotlin.test.* +import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds import kotlin.time.TestTimeSource @@ -51,9 +52,11 @@ public class S3ExpressCredentialsCacheTest { val timeSource = TestTimeSource() val sessionCredentials = Credentials("superFastAccessKey", "superSecretSecretKey", "s3SessionToken") - val expiringSessionCredentials = ExpiringValue(sessionCredentials, timeSource.markNow()) + // credentials expire in 1 minute 1 second, just outside the refresh buffer + val expiringSessionCredentials = ExpiringValue(sessionCredentials, timeSource.markNow() + 1.minutes + 1.seconds) assertFalse(expiringSessionCredentials.isExpired) + timeSource += 1.seconds assertTrue(expiringSessionCredentials.isExpired) } From 898430d268ed8f11efc284e9a6b6339067ed0072 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 20 Feb 2024 11:00:23 -0500 Subject: [PATCH 095/145] Fix application of SigV4 S3 Express auth scheme --- .../s3/express/S3ExpressIntegration.kt | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 80812e86845..a717420b9ff 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -10,21 +10,14 @@ import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration -import software.amazon.smithy.kotlin.codegen.model.buildSymbol -import software.amazon.smithy.kotlin.codegen.model.expectShape -import software.amazon.smithy.kotlin.codegen.model.getTrait -import software.amazon.smithy.kotlin.codegen.model.hasTrait +import software.amazon.smithy.kotlin.codegen.model.* import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware import software.amazon.smithy.kotlin.codegen.utils.dq import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model -import software.amazon.smithy.model.shapes.OperationShape -import software.amazon.smithy.model.shapes.ServiceShape -import software.amazon.smithy.model.shapes.StructureShape -import software.amazon.smithy.model.traits.AuthTrait -import software.amazon.smithy.model.traits.HttpChecksumRequiredTrait -import software.amazon.smithy.model.traits.HttpHeaderTrait +import software.amazon.smithy.model.shapes.* +import software.amazon.smithy.model.traits.* import software.amazon.smithy.model.transform.ModelTransformer /** @@ -41,24 +34,27 @@ class S3ExpressIntegration : KotlinIntegration { /** * Add a synthetic SigV4 S3 Express auth trait */ - override fun preprocessModel(model: Model, settings: KotlinSettings): Model = - ModelTransformer.create().mapShapes(model) { shape -> - when { - shape.isServiceShape -> { - val builder = (shape as ServiceShape).toBuilder() + override fun preprocessModel(model: Model, settings: KotlinSettings): Model { + val transformer = ModelTransformer.create() - builder.addTrait(SigV4S3ExpressAuthTrait()) + // AuthIndex.getAuthSchemes looks for shapes with an AuthDefinitionTrait, so make one for SigV4 S3Express + val authDefinitionTrait = AuthDefinitionTrait.builder().addTrait(SigV4S3ExpressAuthTrait.ID).build() + val sigV4S3ExpressAuthShape = StructureShape.builder() + .addTrait(authDefinitionTrait) + .id(SigV4S3ExpressAuthTrait.ID) + .build() - val authTrait = - AuthTrait(mutableSetOf(SigV4S3ExpressAuthTrait.ID) + shape.expectTrait(AuthTrait::class.java).valueSet) - builder.addTrait(authTrait) + val serviceShape = settings.getService(model) + val serviceShapeBuilder = serviceShape.toBuilder() - builder.build() - } + serviceShapeBuilder.addTrait(SigV4S3ExpressAuthTrait()) - else -> shape - } - } + val authTrait = AuthTrait(mutableSetOf(SigV4S3ExpressAuthTrait.ID) + serviceShape.expectTrait(AuthTrait::class.java).valueSet) + serviceShapeBuilder.addTrait(authTrait) + + // Add new shape and update service shape + return transformer.replaceShapes(model, listOf(sigV4S3ExpressAuthShape, serviceShapeBuilder.build())) + } override fun customizeMiddleware(ctx: ProtocolGenerator.GenerationContext, resolved: List) = resolved + listOf( From 89eeae8d10bff63019c18cdf7cf72f4a5c7e21d2 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 20 Feb 2024 14:56:36 -0500 Subject: [PATCH 096/145] Add tests --- .../DefaultS3ExpressCredentialsProvider.kt | 74 ++++--- .../s3/express/S3ExpressCredentialsCache.kt | 2 + ...DefaultS3ExpressCredentialsProviderTest.kt | 193 ++++++++++++++++++ 3 files changed, 236 insertions(+), 33 deletions(-) create mode 100644 services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 2887fefc0c9..8677635453f 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -32,7 +32,7 @@ internal val REFRESH_BUFFER = 1.minutes /** * How long to wait between cache refresh attempts if no [Credentials] are in the cache */ -private val DEFAULT_REFRESH_PERIOD = 3.minutes +internal val DEFAULT_REFRESH_PERIOD = 3.minutes private const val CREDENTIALS_PROVIDER_NAME = "DefaultS3ExpressCredentialsProvider" @@ -41,14 +41,15 @@ private const val CREDENTIALS_PROVIDER_NAME = "DefaultS3ExpressCredentialsProvid * @param timeSource the time source to use. defaults to [TimeSource.Monotonic] * @param clock the clock to use. defaults to [Clock.System]. note: the clock is only used to get an initial [Duration] * until credentials expiration. + * @param credentialsCache an [S3ExpressCredentialsCache] to be used for caching session credentials, defaults to + * [S3ExpressCredentialsCache]. */ internal class DefaultS3ExpressCredentialsProvider( private val timeSource: TimeSource = TimeSource.Monotonic, private val clock: Clock = Clock.System, + private val credentialsCache: S3ExpressCredentialsCache = S3ExpressCredentialsCache(), ) : S3ExpressCredentialsProvider, SdkManagedBase(), CoroutineScope { private lateinit var client: S3Client - private val credentialsCache = S3ExpressCredentialsCache() - override val coroutineContext: CoroutineContext = Job() + CoroutineName(CREDENTIALS_PROVIDER_NAME) init { @@ -63,7 +64,7 @@ internal class DefaultS3ExpressCredentialsProvider( val key = S3ExpressCredentialsCacheKey(attributes[S3Attributes.Bucket], client.config.credentialsProvider.resolve(attributes)) return credentialsCache.get(key)?.expiringCredentials?.takeIf { !it.isExpired }?.value - ?: createSessionCredentials(key.bucket).also { credentialsCache.put(key, S3ExpressCredentialsCacheValue(it, usedSinceLastRefresh = true)) }.value + ?: createSessionCredentials(key.bucket, client).also { credentialsCache.put(key, S3ExpressCredentialsCacheValue(it, usedSinceLastRefresh = true)) }.value } override fun close() = coroutineContext.cancel(null) @@ -72,7 +73,7 @@ internal class DefaultS3ExpressCredentialsProvider( * Attempt to refresh the credentials in the cache. A refresh is initiated when the `nextRefresh` time has been reached, * which is either `DEFAULT_REFRESH_PERIOD` or the soonest credentials expiration time (minus a buffer), whichever comes first. */ - private suspend fun refresh() { + internal suspend fun refresh() { while (isActive) { if (!this::client.isInitialized) { delay(5.seconds) @@ -92,43 +93,50 @@ internal class DefaultS3ExpressCredentialsProvider( } // Mark all credentials as not used since last refresh - entries.forEach { - it.value.usedSinceLastRefresh = false - } + entries.forEach { it.value.usedSinceLastRefresh = false } // Refresh any credentials which are already expired - val expiredEntries = entries.filter { it.value.expiringCredentials.isExpired } - - supervisorScope { - expiredEntries.forEach { entry -> - logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..." } - - try { - val refreshed = async { createSessionCredentials(entry.key.bucket) }.await() - credentialsCache.put(entry.key, S3ExpressCredentialsCacheValue(refreshed, usedSinceLastRefresh = false)) - } catch (e: Exception) { - logger.warn(e) { "Failed to refresh credentials for ${entry.key.bucket}" } - } - } - } + refreshExpiredEntries(entries.filter { it.value.expiringCredentials.isExpired }) - // Find the next expiring credentials, sleep until then - val nextExpiringEntry = entries.maxByOrNull { - // note: `expiresAt` is a future time, which means the `elapsedNow` values are negative - // and count up until expiration at t=0. that's why `maxBy` is used instead of `minBy` - it.value.expiringCredentials.expiresAt.elapsedNow() + // delay until the next refresh + getNextRefreshDuration(entries).also { + logger.debug { "Completed credentials refresh, next attempt in $it" } + delay(it) } + } + } - val delayDuration = nextExpiringEntry - ?.let { timeSource.markNow().until(it.value.expiringCredentials.expiresAt) } - ?: DEFAULT_REFRESH_PERIOD + /** + * Attempt to refresh expired credentials cache entries. + */ + internal suspend fun refreshExpiredEntries(expiredEntries: Collection, client: S3Client = this.client) { + supervisorScope { + expiredEntries.forEach { entry -> + logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..." } + + try { + val refreshed = async { createSessionCredentials(entry.key.bucket, client) }.await() + credentialsCache.put(entry.key, S3ExpressCredentialsCacheValue(refreshed, usedSinceLastRefresh = false)) + } catch (e: Exception) { + logger.warn(e) { "Failed to refresh credentials for ${entry.key.bucket}" } + } + } + } + } - logger.debug { "Completed credentials refresh, next attempt in $delayDuration" } - delay(delayDuration) + internal fun getNextRefreshDuration(entries: Collection): Duration { + val nextExpiringEntry = entries.maxByOrNull { + // note: `expiresAt` is a future time, which means the `elapsedNow` values are negative + // and count up until expiration at t=0. that's why `maxBy` is used instead of `minBy` + it.value.expiringCredentials.expiresAt.elapsedNow() } + + return nextExpiringEntry?.let { + timeSource.markNow().until(it.value.expiringCredentials.expiresAt) - REFRESH_BUFFER + } ?: DEFAULT_REFRESH_PERIOD } - private suspend fun createSessionCredentials(bucket: String): ExpiringValue { + internal suspend fun createSessionCredentials(bucket: String, client: S3Client): ExpiringValue { val credentials = client.createSession { this.bucket = bucket }.credentials!! val expirationTimeMark = timeSource.markNow() + clock.now().until(credentials.expiration) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index 51f6a7f9378..63a0689deb6 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -42,3 +42,5 @@ internal data class S3ExpressCredentialsCacheValue( internal data class ExpiringValue (val value: T, val expiresAt: TimeMark) internal val ExpiringValue.isExpired: Boolean get() = (expiresAt - REFRESH_BUFFER).hasPassedNow() + +internal typealias S3ExpressCredentialsCacheEntry = Map.Entry diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt new file mode 100644 index 00000000000..e3dd9e17b15 --- /dev/null +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt @@ -0,0 +1,193 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.services.s3.express + +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CreateSessionRequest +import aws.sdk.kotlin.services.s3.model.CreateSessionResponse +import aws.sdk.kotlin.services.s3.model.SessionCredentials +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.io.use +import aws.smithy.kotlin.runtime.time.ManualClock +import kotlinx.coroutines.test.runTest +import kotlin.test.* +import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds +import kotlin.time.TestTimeSource +import kotlin.time.TimeMark + +class DefaultS3ExpressCredentialsProviderTest { + @Test + fun testGetNextRefreshDuration() = runTest { + val timeSource = TestTimeSource() + + val entries = mutableListOf( + getEntry(timeSource.markNow() + 5.minutes), + getEntry(timeSource.markNow() + 6.minutes), + getEntry(timeSource.markNow() + 7.minutes), + ) + + timeSource += 2.minutes + + DefaultS3ExpressCredentialsProvider(timeSource).use { + // earliest expiration in 3 minutes, minus a 1 minute buffer = 2 minutes + assertEquals(2.minutes, it.getNextRefreshDuration(entries)) + + // Empty list of entries should result in the default refresh period + assertEquals(DEFAULT_REFRESH_PERIOD, it.getNextRefreshDuration(listOf())) + } + } + + @Test + fun testCreateSessionCredentials() = runTest { + val timeSource = TestTimeSource() + val clock = ManualClock() + + val expectedCredentials = SessionCredentials { + accessKeyId = "access" + secretAccessKey = "secret" + sessionToken = "session" + expiration = clock.now() + 5.minutes + } + + val client = TestS3Client(expectedCredentials) + + DefaultS3ExpressCredentialsProvider(timeSource, clock).use { provider -> + val credentials = provider.createSessionCredentials("bucket", client) + assertFalse(credentials.isExpired) + assertEquals(timeSource.markNow() + 5.minutes, credentials.expiresAt) + } + } + + @Test + fun testRefreshExpiredEntries() = runTest { + val timeSource = TestTimeSource() + val clock = ManualClock() + val cache = S3ExpressCredentialsCache() + val expectedCredentials = SessionCredentials { + accessKeyId = "access" + secretAccessKey = "secret" + sessionToken = "session" + expiration = clock.now() + 5.minutes + } + + val expiredEntries = listOf( + getEntry(timeSource.markNow(), bootstrapCredentials = Credentials("1", "1", "1")), + getEntry(timeSource.markNow() + 30.seconds, bootstrapCredentials = Credentials("2", "2", "2")), + getEntry(timeSource.markNow() + 1.minutes, bootstrapCredentials = Credentials("3", "3", "3")), + getEntry(timeSource.markNow() + 1.minutes + 30.seconds, bootstrapCredentials = Credentials("4", "4", "4")), + getEntry(timeSource.markNow() + 2.minutes, bootstrapCredentials = Credentials("5", "5", "5")), + getEntry(timeSource.markNow() + 2.minutes + 30.seconds, bootstrapCredentials = Credentials("6", "6", "6")), + ) + timeSource += 3.minutes + assertTrue(expiredEntries.all { it.value.expiringCredentials.isExpired }) // validate all entries are now expired + + val testS3Client = TestS3Client(expectedCredentials) + + DefaultS3ExpressCredentialsProvider(timeSource, clock, cache).use { provider -> + provider.refreshExpiredEntries(expiredEntries, testS3Client) + } + + val refreshedEntries = cache.entries + assertFalse(refreshedEntries.any { it.value.expiringCredentials.isExpired }) // none of the entries are expired + assertFalse(refreshedEntries.any { it.value.usedSinceLastRefresh }) // none of the entries have been used since the last refresh + } + + @Test + fun testRefreshExpiredEntriesHandlesFailures() = runTest { + val timeSource = TestTimeSource() + val clock = ManualClock() + val cache = S3ExpressCredentialsCache() + + val expectedCredentials = SessionCredentials { + accessKeyId = "access" + secretAccessKey = "secret" + sessionToken = "session" + expiration = clock.now() + 5.minutes + } + val testS3Client = TestS3Client(expectedCredentials, throwExceptionOnBucket = "ExceptionBucket") + + val expiredEntries = listOf( + getEntry(timeSource.markNow() + 30.seconds, bucket = "ExceptionBucket", bootstrapCredentials = Credentials("1", "1", "1")), + getEntry(timeSource.markNow() + 1.minutes, bucket = "SuccessfulBucket", bootstrapCredentials = Credentials("2", "2", "2")), + getEntry(timeSource.markNow() + 1.minutes + 30.seconds, bucket = "SuccessfulBucket", bootstrapCredentials = Credentials("3", "3", "3")), + getEntry(timeSource.markNow() + 2.minutes, bucket = "ExceptionBucket", bootstrapCredentials = Credentials("4", "4", "4")), + getEntry(timeSource.markNow(), bucket = "SuccessfulBucket", bootstrapCredentials = Credentials("5", "5", "5")), + ) + expiredEntries.forEach { cache.put(it.key, it.value) } + assertEquals(5, cache.size) + + timeSource += 3.minutes + assertTrue(expiredEntries.all { it.value.expiringCredentials.isExpired }) // all entries are now expired + + DefaultS3ExpressCredentialsProvider(timeSource, clock, cache).use { provider -> + provider.refreshExpiredEntries(expiredEntries, testS3Client) + } + + val refreshedEntries = cache.entries + assertEquals(5, refreshedEntries.size) // no entries were removed + + // two entries failed to refresh, they are still expired. + assertEquals( + 2, + refreshedEntries + .filter { it.value.expiringCredentials.isExpired } + .size, + ) + assertTrue( + refreshedEntries + .filter { it.value.expiringCredentials.isExpired } + .all { it.key.bucket == "ExceptionBucket" }, + ) + + // three entries successfully refreshed and are no longer expired + assertEquals( + 3, + refreshedEntries + .filter { !it.value.expiringCredentials.isExpired } + .size, + ) + assertTrue( + refreshedEntries + .filter { !it.value.expiringCredentials.isExpired } + .all { it.key.bucket == "SuccessfulBucket" }, + ) + } + + /** + * Get an instance of [Map.Entry] using the given [expiration], + * [bucket], and optional [bootstrapCredentials] and [sessionCredentials]. + */ + private fun getEntry( + expiration: TimeMark, + bucket: String = "bucket", + bootstrapCredentials: Credentials = Credentials(accessKeyId = "accessKeyId", secretAccessKey = "secretAccessKey", sessionToken = "sessionToken"), + sessionCredentials: Credentials = Credentials(accessKeyId = "s3AccessKeyId", secretAccessKey = "s3SecretAccessKey", sessionToken = "s3SessionToken"), + ): S3ExpressCredentialsCacheEntry = mapOf( + S3ExpressCredentialsCacheKey(bucket, bootstrapCredentials) to S3ExpressCredentialsCacheValue(ExpiringValue(sessionCredentials, expiration)), + ).entries.first() + + /** + * A test S3Client used to mock calls to s3:CreateSession. + * @param expectedCredentials the expected session credentials returned from s3:CreateSession + * @param client the base S3 client used to implement other operations, though they are unused. + * @param throwExceptionOnBucket an optional bucket name, which when specified and present in the [CreateSessionRequest], will + * cause the client to throw an exception instead of returning credentials. Used for testing s3:CreateSession failures. + */ + private class TestS3Client( + val expectedCredentials: SessionCredentials, + val client: S3Client = S3Client { }, + val throwExceptionOnBucket: String? = null, + ) : S3Client by client { + override suspend fun createSession(input: CreateSessionRequest): CreateSessionResponse { + throwExceptionOnBucket?.let { + if (input.bucket == it) { + throw Exception("Failed to create session credentials for bucket: $throwExceptionOnBucket") + } + } + return CreateSessionResponse { credentials = expectedCredentials } + } + } +} From 7db188d4979609e8928196861d5c94f21277f391 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 20 Feb 2024 15:09:33 -0500 Subject: [PATCH 097/145] Update KDocs --- .../customization/s3/express/S3ExpressIntegration.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index a717420b9ff..d9b6740bc87 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -22,8 +22,8 @@ import software.amazon.smithy.model.transform.ModelTransformer /** * An integration which handles codegen for S3 Express, such as: - * 1. Configure auth scheme (auth scheme ID, auth option, identity provider for auth scheme) - * 2. Add S3Client and Bucket to execution context (required for the S3 Express credentials cache key) + * 1. Configure auth scheme by applying a synthetic shape and trait + * 2. Add ExpressClient and Bucket to execution context * 3. Override checksums to use CRC32 instead of MD5 * 4. Disable all checksums for s3:UploadPart */ @@ -32,12 +32,12 @@ class S3ExpressIntegration : KotlinIntegration { model.expectShape(settings.service).isS3 /** - * Add a synthetic SigV4 S3 Express auth trait + * Add a synthetic SigV4 S3 Express auth trait and shape */ override fun preprocessModel(model: Model, settings: KotlinSettings): Model { val transformer = ModelTransformer.create() - // AuthIndex.getAuthSchemes looks for shapes with an AuthDefinitionTrait, so make one for SigV4 S3Express + // AuthIndex.getAuthSchemes looks for shapes with an AuthDefinitionTrait, so need to make one for SigV4 S3Express val authDefinitionTrait = AuthDefinitionTrait.builder().addTrait(SigV4S3ExpressAuthTrait.ID).build() val sigV4S3ExpressAuthShape = StructureShape.builder() .addTrait(authDefinitionTrait) @@ -52,7 +52,7 @@ class S3ExpressIntegration : KotlinIntegration { val authTrait = AuthTrait(mutableSetOf(SigV4S3ExpressAuthTrait.ID) + serviceShape.expectTrait(AuthTrait::class.java).valueSet) serviceShapeBuilder.addTrait(authTrait) - // Add new shape and update service shape + // Add the new shape and update the service shape's AuthTrait return transformer.replaceShapes(model, listOf(sigV4S3ExpressAuthShape, serviceShapeBuilder.build())) } From 344a3ee316efbf21f9c39a9ac57e5e5a50631b2a Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 20 Feb 2024 15:34:10 -0500 Subject: [PATCH 098/145] Update KDocs --- .../services/s3/express/DefaultS3ExpressCredentialsProvider.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 8677635453f..c5bb4e09f6c 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -98,7 +98,7 @@ internal class DefaultS3ExpressCredentialsProvider( // Refresh any credentials which are already expired refreshExpiredEntries(entries.filter { it.value.expiringCredentials.isExpired }) - // delay until the next refresh + // Delay until the next refresh getNextRefreshDuration(entries).also { logger.debug { "Completed credentials refresh, next attempt in $it" } delay(it) From 19f3e7e083ad77afdbc0085022863ff36d88e27a Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 20 Feb 2024 15:59:44 -0500 Subject: [PATCH 099/145] Relocate SigV4S3ExpressAuthScheme to express package --- .../s3/express/SigV4S3ExpressAuthSchemeIntegration.kt | 6 +++--- .../services/s3/{ => express}/SigV4S3ExpressAuthScheme.kt | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) rename services/s3/common/src/aws/sdk/kotlin/services/s3/{ => express}/SigV4S3ExpressAuthScheme.kt (95%) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index 3b8c8fb795c..44f0cec6024 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -55,12 +55,12 @@ class SigV4S3ExpressAuthSchemeIntegration : KotlinIntegration { internal val sigV4S3ExpressSymbol = buildSymbol { name = "sigV4S3Express" - namespace = "aws.sdk.kotlin.services.s3" + namespace = "aws.sdk.kotlin.services.s3.express" } internal val SigV4S3ExpressAuthSchemeSymbol = buildSymbol { name = "SigV4S3ExpressAuthScheme" - namespace = "aws.sdk.kotlin.services.s3" + namespace = "aws.sdk.kotlin.services.s3.express" } private object SigV4S3ExpressEndpointCustomization : EndpointCustomization { @@ -73,7 +73,7 @@ open class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { override val authSchemeId: ShapeId = ShapeId.from("aws.auth#sigv4s3express") override val authSchemeIdSymbol: Symbol = buildSymbol { - name = "AuthSchemeId.AwsSigV4S3Express" + name = "AuthSchemeId(\"aws.auth#sigv4s3express\")" val ref = RuntimeTypes.Auth.Identity.AuthSchemeId objectRef = ref namespace = ref.namespace diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/SigV4S3ExpressAuthScheme.kt similarity index 95% rename from services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt rename to services/s3/common/src/aws/sdk/kotlin/services/s3/express/SigV4S3ExpressAuthScheme.kt index 33c78d74795..53df1abb334 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/SigV4S3ExpressAuthScheme.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/SigV4S3ExpressAuthScheme.kt @@ -2,7 +2,7 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0 */ -package aws.sdk.kotlin.services.s3 +package aws.sdk.kotlin.services.s3.express import aws.sdk.kotlin.services.s3.express.S3ExpressHttpSigner import aws.smithy.kotlin.runtime.InternalApi @@ -15,7 +15,6 @@ import aws.smithy.kotlin.runtime.http.auth.AwsHttpSigner import aws.smithy.kotlin.runtime.http.auth.HttpSigner import aws.smithy.kotlin.runtime.http.auth.sigV4 -// FIXME Had trouble moving this to `express` package because this val became unresolved symbol public val AuthSchemeId.Companion.AwsSigV4S3Express: AuthSchemeId get() = AuthSchemeId("aws.auth#sigv4s3express") From 14ef41141b2b4ebcc0257a331267a54ee9c20de0 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 20 Feb 2024 16:03:55 -0500 Subject: [PATCH 100/145] ktlint --- .../sdk/kotlin/services/s3/express/SigV4S3ExpressAuthScheme.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/SigV4S3ExpressAuthScheme.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/SigV4S3ExpressAuthScheme.kt index 53df1abb334..c14502ab2a3 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/SigV4S3ExpressAuthScheme.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/SigV4S3ExpressAuthScheme.kt @@ -4,7 +4,6 @@ */ package aws.sdk.kotlin.services.s3.express -import aws.sdk.kotlin.services.s3.express.S3ExpressHttpSigner import aws.smithy.kotlin.runtime.InternalApi import aws.smithy.kotlin.runtime.auth.AuthOption import aws.smithy.kotlin.runtime.auth.AuthSchemeId From e6c65b93dd2bec22979a07d0169ea7ac903c624b Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 09:28:33 -0500 Subject: [PATCH 101/145] remove extra space --- .../sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index 63a0689deb6..747795c66f3 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -39,7 +39,7 @@ internal data class S3ExpressCredentialsCacheValue( /** * A value with an expiration [TimeMark] */ -internal data class ExpiringValue (val value: T, val expiresAt: TimeMark) +internal data class ExpiringValue(val value: T, val expiresAt: TimeMark) internal val ExpiringValue.isExpired: Boolean get() = (expiresAt - REFRESH_BUFFER).hasPassedNow() From 758e92e4db1fdbf1fee935bf0a4de618626f5667 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 09:28:52 -0500 Subject: [PATCH 102/145] update `CredentialsProvider` -> `CloseableCredentialsProvider` --- .../services/s3/express/S3ExpressCredentialsProvider.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt index df54698e0fd..3b32ff9fc92 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt @@ -4,12 +4,12 @@ */ package aws.sdk.kotlin.services.s3.express -import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvider +import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider /** * A credentials provider used for making requests to S3 Express One Zone directory buckets. */ -public interface S3ExpressCredentialsProvider : CloseableCredentialsProvider { +public interface S3ExpressCredentialsProvider : CredentialsProvider { public companion object { /** * Create an instance of the default [S3ExpressCredentialsProvider] implementation From 60bef737463ce9dcd67ef48206e1c791be7c6087 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 09:32:21 -0500 Subject: [PATCH 103/145] Replace class with typealias to `LruCache` --- .../s3/express/S3ExpressCredentialsCache.kt | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index 747795c66f3..08c6b146c34 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -10,21 +10,8 @@ import kotlin.time.TimeMark private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 -internal class S3ExpressCredentialsCache { - private val lru = LruCache(DEFAULT_S3_EXPRESS_CACHE_SIZE) - - suspend fun get(key: S3ExpressCredentialsCacheKey): S3ExpressCredentialsCacheValue? = lru.get(key) - - suspend fun put(key: S3ExpressCredentialsCacheKey, value: S3ExpressCredentialsCacheValue) = lru.put(key, value) - - suspend fun remove(key: S3ExpressCredentialsCacheKey): S3ExpressCredentialsCacheValue? = lru.remove(key) - - public val size: Int - get() = lru.size - - public val entries: Set> - get() = lru.entries -} +internal typealias S3ExpressCredentialsCache = LruCache +internal fun S3ExpressCredentialsCache() = S3ExpressCredentialsCache(DEFAULT_S3_EXPRESS_CACHE_SIZE) internal data class S3ExpressCredentialsCacheKey( val bucket: String, From e544637f642084435ba585f829d61aff15be3d33 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 09:32:34 -0500 Subject: [PATCH 104/145] Implement CloseableCredentialsProvider --- .../services/s3/express/DefaultS3ExpressCredentialsProvider.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index c5bb4e09f6c..825f88a2c6a 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -8,6 +8,7 @@ import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.S3Attributes import aws.sdk.kotlin.services.s3.S3Client import aws.smithy.kotlin.runtime.ExperimentalApi +import aws.smithy.kotlin.runtime.auth.awscredentials.CloseableCredentialsProvider import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.collections.get @@ -48,7 +49,7 @@ internal class DefaultS3ExpressCredentialsProvider( private val timeSource: TimeSource = TimeSource.Monotonic, private val clock: Clock = Clock.System, private val credentialsCache: S3ExpressCredentialsCache = S3ExpressCredentialsCache(), -) : S3ExpressCredentialsProvider, SdkManagedBase(), CoroutineScope { +) : S3ExpressCredentialsProvider, CloseableCredentialsProvider, SdkManagedBase(), CoroutineScope { private lateinit var client: S3Client override val coroutineContext: CoroutineContext = Job() + CoroutineName(CREDENTIALS_PROVIDER_NAME) From 1531e732dd81a8fa6a9848ee34f03c8f9d9d71ed Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 09:36:51 -0500 Subject: [PATCH 105/145] Use `ComparableTimeMark` --- .../express/DefaultS3ExpressCredentialsProvider.kt | 12 +++--------- .../services/s3/express/S3ExpressCredentialsCache.kt | 3 ++- .../DefaultS3ExpressCredentialsProviderTest.kt | 4 ++-- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 825f88a2c6a..257e3375057 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -22,7 +22,6 @@ import kotlin.coroutines.CoroutineContext import kotlin.time.Duration import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds -import kotlin.time.TimeMark import kotlin.time.TimeSource /** @@ -46,7 +45,7 @@ private const val CREDENTIALS_PROVIDER_NAME = "DefaultS3ExpressCredentialsProvid * [S3ExpressCredentialsCache]. */ internal class DefaultS3ExpressCredentialsProvider( - private val timeSource: TimeSource = TimeSource.Monotonic, + private val timeSource: TimeSource.WithComparableMarks = TimeSource.Monotonic, private val clock: Clock = Clock.System, private val credentialsCache: S3ExpressCredentialsCache = S3ExpressCredentialsCache(), ) : S3ExpressCredentialsProvider, CloseableCredentialsProvider, SdkManagedBase(), CoroutineScope { @@ -133,7 +132,7 @@ internal class DefaultS3ExpressCredentialsProvider( } return nextExpiringEntry?.let { - timeSource.markNow().until(it.value.expiringCredentials.expiresAt) - REFRESH_BUFFER + it.value.expiringCredentials.expiresAt - timeSource.markNow() - REFRESH_BUFFER } ?: DEFAULT_REFRESH_PERIOD } @@ -159,9 +158,4 @@ internal class DefaultS3ExpressCredentialsProvider( } else { TelemetryProvider.None.loggerProvider.getLogger() } -} - -/** - * Get the [Duration] between [this] TimeMark and an [other] TimeMark - */ -internal fun TimeMark.until(other: TimeMark): Duration = (this.elapsedNow().absoluteValue - other.elapsedNow().absoluteValue).absoluteValue +} \ No newline at end of file diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index 08c6b146c34..ac4f1e87397 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -6,6 +6,7 @@ package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.collections.LruCache +import kotlin.time.ComparableTimeMark import kotlin.time.TimeMark private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 @@ -26,7 +27,7 @@ internal data class S3ExpressCredentialsCacheValue( /** * A value with an expiration [TimeMark] */ -internal data class ExpiringValue(val value: T, val expiresAt: TimeMark) +internal data class ExpiringValue(val value: T, val expiresAt: ComparableTimeMark) internal val ExpiringValue.isExpired: Boolean get() = (expiresAt - REFRESH_BUFFER).hasPassedNow() diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt index e3dd9e17b15..c66d190d5d1 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt @@ -13,10 +13,10 @@ import aws.smithy.kotlin.runtime.io.use import aws.smithy.kotlin.runtime.time.ManualClock import kotlinx.coroutines.test.runTest import kotlin.test.* +import kotlin.time.ComparableTimeMark import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds import kotlin.time.TestTimeSource -import kotlin.time.TimeMark class DefaultS3ExpressCredentialsProviderTest { @Test @@ -161,7 +161,7 @@ class DefaultS3ExpressCredentialsProviderTest { * [bucket], and optional [bootstrapCredentials] and [sessionCredentials]. */ private fun getEntry( - expiration: TimeMark, + expiration: ComparableTimeMark, bucket: String = "bucket", bootstrapCredentials: Credentials = Credentials(accessKeyId = "accessKeyId", secretAccessKey = "secretAccessKey", sessionToken = "sessionToken"), sessionCredentials: Credentials = Credentials(accessKeyId = "s3AccessKeyId", secretAccessKey = "s3SecretAccessKey", sessionToken = "s3SessionToken"), From 1f566f17d03295874cad289e2b515db2758b0aac Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 09:38:37 -0500 Subject: [PATCH 106/145] remove `open` --- .../s3/express/SigV4S3ExpressAuthSchemeIntegration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index 44f0cec6024..51ab8e9b89d 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -69,7 +69,7 @@ private object SigV4S3ExpressEndpointCustomization : EndpointCustomization { ) } -open class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { +class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { override val authSchemeId: ShapeId = ShapeId.from("aws.auth#sigv4s3express") override val authSchemeIdSymbol: Symbol = buildSymbol { From 70f7de5bc19384f77ca1e9bad37d4f0634c85823 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 10:04:44 -0500 Subject: [PATCH 107/145] Add a comment about running after `SigV4AuthSchemeIntegration` --- .../s3/express/SigV4S3ExpressAuthSchemeIntegration.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index 51ab8e9b89d..76a16616ed6 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -36,7 +36,8 @@ import java.util.* * Register support for the `sigv4-s3express` auth scheme. */ class SigV4S3ExpressAuthSchemeIntegration : KotlinIntegration { - override val order: Byte = -50 + // Needs to run after `SigV4AuthSchemeIntegration` + override val order: Byte = -51 override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = model.expectShape(settings.service).isS3 @@ -81,7 +82,7 @@ class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { } override fun identityProviderAdapterExpression(writer: KotlinWriter) { - writer.write("config.#L", ClientConfigIntegration.ExpressCredentialsProvider.propertyName) + writer.write("config.#L", S3ExpressIntegration.ExpressCredentialsProvider.propertyName) } override fun authSchemeProviderInstantiateAuthOptionExpr( From 0cae04530e7f5524a168bff73fad798d574d94e7 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 10:05:03 -0500 Subject: [PATCH 108/145] Relocate S3 Express client config properties --- .../s3/ClientConfigIntegration.kt | 32 ---------- .../s3/express/S3ExpressIntegration.kt | 60 +++++++++++++++---- .../endpoints/BindAwsEndpointBuiltins.kt | 3 +- 3 files changed, 52 insertions(+), 43 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index cb2f8edb9ba..0bc4fbd9403 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -68,36 +68,6 @@ class ClientConfigIntegration : KotlinIntegration { useSymbolWithNullableBuilder(KotlinTypes.Boolean, "true") documentation = "Flag to enable [aws-chunked](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html) content encoding." } - - val DisableExpressSessionAuth: ConfigProperty = ConfigProperty { - name = "disableS3ExpressSessionAuth" - useSymbolWithNullableBuilder(KotlinTypes.Boolean, "false") - documentation = """ - Flag to disable S3 Express One Zone's bucket-level session authentication method. - """.trimIndent() - } - - val ExpressCredentialsProvider: ConfigProperty = ConfigProperty { - name = "expressCredentialsProvider" - symbol = buildSymbol { - name = "S3ExpressCredentialsProvider" - nullable = false - namespace = "aws.sdk.kotlin.services.s3.express" - } - documentation = """ - Credentials provider to be used for making requests to S3 Express. - """.trimIndent() - - propertyType = ConfigPropertyType.Custom( - render = { _, writer -> - writer.write("public val #1L: #2T = builder.#1L ?: #2T.default()", name, symbol) - }, - renderBuilder = { prop, writer -> - prop.documentation?.let(writer::dokka) - writer.write("public var #L: #T? = null", name, symbol) - }, - ) - } } override fun preprocessModel(model: Model, settings: KotlinSettings): Model { @@ -124,8 +94,6 @@ class ClientConfigIntegration : KotlinIntegration { UseArnRegionProp, DisableMrapProp, EnableAwsChunked, - DisableExpressSessionAuth, - ExpressCredentialsProvider, ) override val sectionWriters: List diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index d9b6740bc87..1b399adc328 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -10,9 +10,12 @@ import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +import software.amazon.smithy.kotlin.codegen.lang.KotlinTypes import software.amazon.smithy.kotlin.codegen.model.* import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty +import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.kotlin.codegen.utils.dq import software.amazon.smithy.kotlin.codegen.utils.getOrNull import software.amazon.smithy.model.Model @@ -28,6 +31,38 @@ import software.amazon.smithy.model.transform.ModelTransformer * 4. Disable all checksums for s3:UploadPart */ class S3ExpressIntegration : KotlinIntegration { + companion object { + val DisableExpressSessionAuth: ConfigProperty = ConfigProperty { + name = "disableS3ExpressSessionAuth" + useSymbolWithNullableBuilder(KotlinTypes.Boolean, "false") + documentation = """ + Flag to disable S3 Express One Zone's bucket-level session authentication method. + """.trimIndent() + } + + val ExpressCredentialsProvider: ConfigProperty = ConfigProperty { + name = "expressCredentialsProvider" + symbol = buildSymbol { + name = "S3ExpressCredentialsProvider" + nullable = false + namespace = "aws.sdk.kotlin.services.s3.express" + } + documentation = """ + Credentials provider to be used for making requests to S3 Express. + """.trimIndent() + + propertyType = ConfigPropertyType.Custom( + render = { _, writer -> + writer.write("public val #1L: #2T = builder.#1L ?: #2T.default()", name, symbol) + }, + renderBuilder = { prop, writer -> + prop.documentation?.let(writer::dokka) + writer.write("public var #L: #T? = null", name, symbol) + }, + ) + } + } + override fun enabledForService(model: Model, settings: KotlinSettings) = model.expectShape(settings.service).isS3 @@ -64,6 +99,11 @@ class S3ExpressIntegration : KotlinIntegration { UploadPartDisableChecksum, ) + private val S3AttributesSymbol = buildSymbol { + name = "S3Attributes" + namespace = "aws.sdk.kotlin.services.s3" + } + private val AddClientToExecutionContext = object : ProtocolMiddleware { override val name: String = "AddClientToExecutionContext" @@ -71,11 +111,7 @@ class S3ExpressIntegration : KotlinIntegration { ctx.model.expectShape(ctx.settings.service).isS3 override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val attributesSymbol = buildSymbol { - name = "S3Attributes" - namespace = "aws.sdk.kotlin.services.s3" - } - writer.write("op.context[#T.ExpressClient] = this", attributesSymbol) + writer.write("op.context[#T.ExpressClient] = this", S3AttributesSymbol) } } @@ -88,11 +124,7 @@ class S3ExpressIntegration : KotlinIntegration { .any { it.memberName == "Bucket" } override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { - val attributesSymbol = buildSymbol { - name = "S3Attributes" - namespace = "aws.sdk.kotlin.services.s3" - } - writer.write("input.bucket?.let { op.context[#T.Bucket] = it }", attributesSymbol) + writer.write("input.bucket?.let { op.context[#T.Bucket] = it }", S3AttributesSymbol) } } @@ -131,6 +163,9 @@ class S3ExpressIntegration : KotlinIntegration { } } + /** + * Disable all checksums for s3:UploadPart + */ private val UploadPartDisableChecksum = object : ProtocolMiddleware { override val name: String = "UploadPartDisableChecksum" @@ -148,4 +183,9 @@ class S3ExpressIntegration : KotlinIntegration { } private val OperationShape.isS3UploadPart: Boolean get() = id.name == "UploadPart" + + override fun additionalServiceConfigProps(ctx: CodegenContext): List = listOf( + DisableExpressSessionAuth, + ExpressCredentialsProvider, + ) } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/BindAwsEndpointBuiltins.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/BindAwsEndpointBuiltins.kt index 46dea048c53..5c418be243e 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/BindAwsEndpointBuiltins.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/endpoints/BindAwsEndpointBuiltins.kt @@ -6,6 +6,7 @@ package aws.sdk.kotlin.codegen.endpoints import aws.sdk.kotlin.codegen.AwsRuntimeTypes import aws.sdk.kotlin.codegen.customization.AccountIdEndpointBuiltinCustomization +import aws.sdk.kotlin.codegen.customization.s3.express.S3ExpressIntegration import software.amazon.smithy.codegen.core.Symbol import software.amazon.smithy.kotlin.codegen.core.CodegenContext import software.amazon.smithy.kotlin.codegen.core.KotlinWriter @@ -90,7 +91,7 @@ fun renderBindAwsBuiltins(ctx: ProtocolGenerator.GenerationContext, writer: Kotl AwsBuiltins.S3_ACCELERATE -> renderBasicConfigBinding(writer, it, S3ClientConfigIntegration.EnableAccelerateProp.propertyName) AwsBuiltins.S3_FORCE_PATH_STYLE -> renderBasicConfigBinding(writer, it, S3ClientConfigIntegration.ForcePathStyleProp.propertyName) AwsBuiltins.S3_DISABLE_MRAP -> renderBasicConfigBinding(writer, it, S3ClientConfigIntegration.DisableMrapProp.propertyName) - AwsBuiltins.S3_DISABLE_EXPRESS_SESSION_AUTH -> renderBasicConfigBinding(writer, it, S3ClientConfigIntegration.DisableExpressSessionAuth.propertyName) + AwsBuiltins.S3_DISABLE_EXPRESS_SESSION_AUTH -> renderBasicConfigBinding(writer, it, S3ExpressIntegration.DisableExpressSessionAuth.propertyName) AwsBuiltins.S3_USE_ARN_REGION -> renderBasicConfigBinding(writer, it, S3ClientConfigIntegration.UseArnRegionProp.propertyName) AwsBuiltins.S3_CONTROL_USE_ARN_REGION -> renderBasicConfigBinding(writer, it, S3ControlClientConfigIntegration.UseArnRegionProp.propertyName) From 674c5214836321eb6ebb620568619c52507c531c Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 10:46:41 -0500 Subject: [PATCH 109/145] ktlint --- .../kotlin/codegen/customization/s3/ClientConfigIntegration.kt | 1 - .../s3/express/SigV4S3ExpressAuthSchemeIntegration.kt | 1 - .../services/s3/express/DefaultS3ExpressCredentialsProvider.kt | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt index 0bc4fbd9403..82fd6388ac1 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/ClientConfigIntegration.kt @@ -14,7 +14,6 @@ import software.amazon.smithy.kotlin.codegen.model.buildSymbol import software.amazon.smithy.kotlin.codegen.model.expectShape import software.amazon.smithy.kotlin.codegen.rendering.ServiceClientGenerator import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigProperty -import software.amazon.smithy.kotlin.codegen.rendering.util.ConfigPropertyType import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.ServiceShape import software.amazon.smithy.model.transform.ModelTransformer diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index 76a16616ed6..4ec6ca4986c 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -4,7 +4,6 @@ */ package aws.sdk.kotlin.codegen.customization.s3.express -import aws.sdk.kotlin.codegen.customization.s3.ClientConfigIntegration import aws.sdk.kotlin.codegen.customization.s3.isS3 import software.amazon.smithy.aws.traits.auth.UnsignedPayloadTrait import software.amazon.smithy.codegen.core.Symbol diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 257e3375057..d8c5bcf4499 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -158,4 +158,4 @@ internal class DefaultS3ExpressCredentialsProvider( } else { TelemetryProvider.None.loggerProvider.getLogger() } -} \ No newline at end of file +} From 0e3177b523cc5e7c0cd1248aea1094375c5049a8 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 14:00:43 -0500 Subject: [PATCH 110/145] Add SFG --- .../s3/express/S3ExpressCredentialsCache.kt | 20 +++++++++++++++++-- .../express/S3ExpressCredentialsCacheTest.kt | 19 +++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index ac4f1e87397..1c1b2616e92 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -6,7 +6,9 @@ package aws.sdk.kotlin.services.s3.express import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.collections.LruCache +import aws.smithy.kotlin.runtime.util.SingleFlightGroup import kotlin.time.ComparableTimeMark +import kotlin.time.Duration import kotlin.time.TimeMark private const val DEFAULT_S3_EXPRESS_CACHE_SIZE: Int = 100 @@ -15,13 +17,25 @@ internal typealias S3ExpressCredentialsCache = LruCache, - var usedSinceLastRefresh: Boolean = false, + /** + * A [SingleFlightGroup] used to de-duplicate asynchronous refresh attempts + */ + val sfg: SingleFlightGroup = SingleFlightGroup(), ) /** @@ -29,6 +43,8 @@ internal data class S3ExpressCredentialsCacheValue( */ internal data class ExpiringValue(val value: T, val expiresAt: ComparableTimeMark) -internal val ExpiringValue.isExpired: Boolean get() = (expiresAt - REFRESH_BUFFER).hasPassedNow() +internal val ExpiringValue.isExpired: Boolean get() = expiresAt.hasPassedNow() + +internal fun ExpiringValue.isExpiringWithin(duration: Duration) = (expiresAt - duration).hasPassedNow() internal typealias S3ExpressCredentialsCacheEntry = Map.Entry diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt index 2ae7da8b203..0ca7608b97d 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt @@ -53,11 +53,24 @@ public class S3ExpressCredentialsCacheTest { val sessionCredentials = Credentials("superFastAccessKey", "superSecretSecretKey", "s3SessionToken") - // credentials expire in 1 minute 1 second, just outside the refresh buffer - val expiringSessionCredentials = ExpiringValue(sessionCredentials, timeSource.markNow() + 1.minutes + 1.seconds) + val expiringSessionCredentials = ExpiringValue(sessionCredentials, timeSource.markNow() + 5.minutes) assertFalse(expiringSessionCredentials.isExpired) - timeSource += 1.seconds + timeSource += 5.minutes + 1.seconds // advance just past the expiration time assertTrue(expiringSessionCredentials.isExpired) } + + @Test + fun testIsWithin() = runTest { + val timeSource = TestTimeSource() + + val sessionCredentials = Credentials("superFastAccessKey", "superSecretSecretKey", "s3SessionToken") + + val expiringSessionCredentials = ExpiringValue(sessionCredentials, timeSource.markNow() + 1.minutes) + assertFalse(expiringSessionCredentials.isExpiringWithin(30.seconds)) + + timeSource += 31.seconds + assertTrue(expiringSessionCredentials.isExpiringWithin(30.seconds)) + + } } From ef4035a5a3ab76fde551b0fd75b06816dc0329b1 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 14:42:53 -0500 Subject: [PATCH 111/145] Refactor to usage-based async refresh --- .../DefaultS3ExpressCredentialsProvider.kt | 147 +++--------- ...DefaultS3ExpressCredentialsProviderTest.kt | 218 +++++++++--------- 2 files changed, 151 insertions(+), 214 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index d8c5bcf4499..913bf93685c 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -13,7 +13,6 @@ import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.collections.get import aws.smithy.kotlin.runtime.io.SdkManagedBase -import aws.smithy.kotlin.runtime.telemetry.TelemetryProvider import aws.smithy.kotlin.runtime.telemetry.logging.getLogger import aws.smithy.kotlin.runtime.time.Clock import aws.smithy.kotlin.runtime.time.until @@ -21,141 +20,71 @@ import kotlinx.coroutines.* import kotlin.coroutines.CoroutineContext import kotlin.time.Duration import kotlin.time.Duration.Companion.minutes -import kotlin.time.Duration.Companion.seconds import kotlin.time.TimeSource /** - * The duration before expiration that [Credentials] are considered expired - */ -internal val REFRESH_BUFFER = 1.minutes - -/** - * How long to wait between cache refresh attempts if no [Credentials] are in the cache - */ -internal val DEFAULT_REFRESH_PERIOD = 3.minutes - -private const val CREDENTIALS_PROVIDER_NAME = "DefaultS3ExpressCredentialsProvider" - -/** - * The default implementation of [S3ExpressCredentialsProvider] + * The default implementation of [S3ExpressCredentialsProvider]. Performs best-effort asynchronous refresh if the + * cached credentials are within their refresh window ([REFRESH_BUFFER] away from expiration) during a call to [resolve]. + * Otherwise, performs synchronous refresh. + * * @param timeSource the time source to use. defaults to [TimeSource.Monotonic] * @param clock the clock to use. defaults to [Clock.System]. note: the clock is only used to get an initial [Duration] * until credentials expiration. * @param credentialsCache an [S3ExpressCredentialsCache] to be used for caching session credentials, defaults to * [S3ExpressCredentialsCache]. + * @param refreshBuffer an optional [Duration] representing the duration before expiration that [Credentials] + * are considered refreshable, defaults to 1 minute. */ internal class DefaultS3ExpressCredentialsProvider( private val timeSource: TimeSource.WithComparableMarks = TimeSource.Monotonic, private val clock: Clock = Clock.System, private val credentialsCache: S3ExpressCredentialsCache = S3ExpressCredentialsCache(), + private val refreshBuffer: Duration = 1.minutes, ) : S3ExpressCredentialsProvider, CloseableCredentialsProvider, SdkManagedBase(), CoroutineScope { private lateinit var client: S3Client - override val coroutineContext: CoroutineContext = Job() + CoroutineName(CREDENTIALS_PROVIDER_NAME) - - init { - launch(coroutineContext) { - refresh() - } - } + override val coroutineContext: CoroutineContext = Job() + CoroutineName("DefaultS3ExpressCredentialsProvider") override suspend fun resolve(attributes: Attributes): Credentials { client = attributes[S3Attributes.ExpressClient] as S3Client val key = S3ExpressCredentialsCacheKey(attributes[S3Attributes.Bucket], client.config.credentialsProvider.resolve(attributes)) - return credentialsCache.get(key)?.expiringCredentials?.takeIf { !it.isExpired }?.value - ?: createSessionCredentials(key.bucket, client).also { credentialsCache.put(key, S3ExpressCredentialsCacheValue(it, usedSinceLastRefresh = true)) }.value - } - - override fun close() = coroutineContext.cancel(null) - - /** - * Attempt to refresh the credentials in the cache. A refresh is initiated when the `nextRefresh` time has been reached, - * which is either `DEFAULT_REFRESH_PERIOD` or the soonest credentials expiration time (minus a buffer), whichever comes first. - */ - internal suspend fun refresh() { - while (isActive) { - if (!this::client.isInitialized) { - delay(5.seconds) - continue - } else if (credentialsCache.size == 0) { - logger.trace { "Cache is empty, waiting..." } - delay(5.seconds) - continue - } - - val entries = credentialsCache.entries - - // Evict any credentials that weren't used since the last refresh - entries.filter { !it.value.usedSinceLastRefresh }.forEach { - logger.debug { "Credentials for ${it.key.bucket} were not used since last refresh, evicting..." } - credentialsCache.remove(it.key) - } - - // Mark all credentials as not used since last refresh - entries.forEach { it.value.usedSinceLastRefresh = false } - - // Refresh any credentials which are already expired - refreshExpiredEntries(entries.filter { it.value.expiringCredentials.isExpired }) - - // Delay until the next refresh - getNextRefreshDuration(entries).also { - logger.debug { "Completed credentials refresh, next attempt in $it" } - delay(it) - } - } - } - - /** - * Attempt to refresh expired credentials cache entries. - */ - internal suspend fun refreshExpiredEntries(expiredEntries: Collection, client: S3Client = this.client) { - supervisorScope { - expiredEntries.forEach { entry -> - logger.debug { "Credentials for ${entry.key.bucket} are expired, refreshing..." } - - try { - val refreshed = async { createSessionCredentials(entry.key.bucket, client) }.await() - credentialsCache.put(entry.key, S3ExpressCredentialsCacheValue(refreshed, usedSinceLastRefresh = false)) - } catch (e: Exception) { - logger.warn(e) { "Failed to refresh credentials for ${entry.key.bucket}" } + return credentialsCache.get(key) + ?.takeIf { !it.expiringCredentials.isExpired } + ?.also { + if (it.expiringCredentials.isExpiringWithin(refreshBuffer)) { + it.sfg.singleFlight { + logger.trace { "Credentials for ${key.bucket} are within their refresh window, performing asynchronous refresh..." } + launch(coroutineContext) { + try { + val sessionCredentials = createSessionCredentials(key.bucket, client) + credentialsCache.put(key, S3ExpressCredentialsCacheValue(sessionCredentials)) + } catch (e: Exception) { + logger.warn(e) { "Asynchronous refresh for ${key.bucket} failed." } + } + + } + } } } - } + ?.expiringCredentials + ?.value + ?: createSessionCredentials(key.bucket, client).also { + credentialsCache.put(key, S3ExpressCredentialsCacheValue(it)) + }.value } - internal fun getNextRefreshDuration(entries: Collection): Duration { - val nextExpiringEntry = entries.maxByOrNull { - // note: `expiresAt` is a future time, which means the `elapsedNow` values are negative - // and count up until expiration at t=0. that's why `maxBy` is used instead of `minBy` - it.value.expiringCredentials.expiresAt.elapsedNow() - } - - return nextExpiringEntry?.let { - it.value.expiringCredentials.expiresAt - timeSource.markNow() - REFRESH_BUFFER - } ?: DEFAULT_REFRESH_PERIOD - } - - internal suspend fun createSessionCredentials(bucket: String, client: S3Client): ExpiringValue { - val credentials = client.createSession { this.bucket = bucket }.credentials!! - val expirationTimeMark = timeSource.markNow() + clock.now().until(credentials.expiration) + override fun close() = coroutineContext.cancel(null) - return ExpiringValue( - Credentials( - accessKeyId = credentials.accessKeyId, - secretAccessKey = credentials.secretAccessKey, - sessionToken = credentials.sessionToken, - expiration = credentials.expiration, - providerName = CREDENTIALS_PROVIDER_NAME, - ), - expirationTimeMark, - ) - } + internal suspend fun createSessionCredentials(bucket: String, client: S3Client): ExpiringValue = + client.createSession { this.bucket = bucket }.credentials!!.let { + ExpiringValue( + Credentials(it.accessKeyId, it.secretAccessKey, it.sessionToken, it.expiration), + expiresAt = timeSource.markNow() + clock.now().until(it.expiration), + ) + } @OptIn(ExperimentalApi::class) - internal val logger get() = if (this::client.isInitialized) { + internal val logger get() = client.config.telemetryProvider.loggerProvider.getLogger() - } else { - TelemetryProvider.None.loggerProvider.getLogger() - } } diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt index c66d190d5d1..86122ea8f4b 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt @@ -4,13 +4,21 @@ */ package aws.sdk.kotlin.services.s3.express +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.services.s3.S3Attributes import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.CreateSessionRequest import aws.sdk.kotlin.services.s3.model.CreateSessionResponse import aws.sdk.kotlin.services.s3.model.SessionCredentials +import aws.sdk.kotlin.services.s3.withConfig import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.io.use +import aws.smithy.kotlin.runtime.operation.ExecutionContext import aws.smithy.kotlin.runtime.time.ManualClock +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import kotlin.test.* import kotlin.time.ComparableTimeMark @@ -20,31 +28,36 @@ import kotlin.time.TestTimeSource class DefaultS3ExpressCredentialsProviderTest { @Test - fun testGetNextRefreshDuration() = runTest { + fun testCreateSessionCredentials() = runTest { val timeSource = TestTimeSource() + val clock = ManualClock() - val entries = mutableListOf( - getEntry(timeSource.markNow() + 5.minutes), - getEntry(timeSource.markNow() + 6.minutes), - getEntry(timeSource.markNow() + 7.minutes), - ) - - timeSource += 2.minutes + val expectedCredentials = SessionCredentials { + accessKeyId = "access" + secretAccessKey = "secret" + sessionToken = "session" + expiration = clock.now() + 5.minutes + } - DefaultS3ExpressCredentialsProvider(timeSource).use { - // earliest expiration in 3 minutes, minus a 1 minute buffer = 2 minutes - assertEquals(2.minutes, it.getNextRefreshDuration(entries)) + val client = TestS3Client(expectedCredentials) - // Empty list of entries should result in the default refresh period - assertEquals(DEFAULT_REFRESH_PERIOD, it.getNextRefreshDuration(listOf())) + DefaultS3ExpressCredentialsProvider(timeSource, clock).use { provider -> + val credentials = provider.createSessionCredentials("bucket", client) + assertFalse(credentials.isExpired) + assertEquals(timeSource.markNow() + 5.minutes, credentials.expiresAt) } } @Test - fun testCreateSessionCredentials() = runTest { + fun testAsyncRefresh() = runTest { val timeSource = TestTimeSource() val clock = ManualClock() + // Entry expires in 30 seconds, refresh buffer is 1 minute. Next `resolve` call should trigger the async refresh + val cache = S3ExpressCredentialsCache() + val entry = getCacheEntry(timeSource.markNow() + 30.seconds) + cache.put(entry.key, entry.value) + val expectedCredentials = SessionCredentials { accessKeyId = "access" secretAccessKey = "secret" @@ -52,20 +65,29 @@ class DefaultS3ExpressCredentialsProviderTest { expiration = clock.now() + 5.minutes } - val client = TestS3Client(expectedCredentials) + val testClient = TestS3Client(expectedCredentials) - DefaultS3ExpressCredentialsProvider(timeSource, clock).use { provider -> - val credentials = provider.createSessionCredentials("bucket", client) - assertFalse(credentials.isExpired) - assertEquals(timeSource.markNow() + 5.minutes, credentials.expiresAt) + DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes).use { provider -> + val attributes = ExecutionContext.build { + this.attributes[S3Attributes.ExpressClient] = testClient + this.attributes[S3Attributes.Bucket] = "bucket" + } + + provider.resolve(attributes) + assertEquals(1, testClient.numCreateSession) } } @Test - fun testRefreshExpiredEntries() = runTest { + fun testAsyncRefreshDebounce() = runTest { val timeSource = TestTimeSource() val clock = ManualClock() + + // Entry expires in 30 seconds, refresh buffer is 1 minute. Next `resolve` call should trigger the async refresh val cache = S3ExpressCredentialsCache() + val entry = getCacheEntry(expiration = timeSource.markNow() + 30.seconds) + cache.put(entry.key, entry.value) + val expectedCredentials = SessionCredentials { accessKeyId = "access" secretAccessKey = "secret" @@ -73,33 +95,34 @@ class DefaultS3ExpressCredentialsProviderTest { expiration = clock.now() + 5.minutes } - val expiredEntries = listOf( - getEntry(timeSource.markNow(), bootstrapCredentials = Credentials("1", "1", "1")), - getEntry(timeSource.markNow() + 30.seconds, bootstrapCredentials = Credentials("2", "2", "2")), - getEntry(timeSource.markNow() + 1.minutes, bootstrapCredentials = Credentials("3", "3", "3")), - getEntry(timeSource.markNow() + 1.minutes + 30.seconds, bootstrapCredentials = Credentials("4", "4", "4")), - getEntry(timeSource.markNow() + 2.minutes, bootstrapCredentials = Credentials("5", "5", "5")), - getEntry(timeSource.markNow() + 2.minutes + 30.seconds, bootstrapCredentials = Credentials("6", "6", "6")), - ) - timeSource += 3.minutes - assertTrue(expiredEntries.all { it.value.expiringCredentials.isExpired }) // validate all entries are now expired - - val testS3Client = TestS3Client(expectedCredentials) - - DefaultS3ExpressCredentialsProvider(timeSource, clock, cache).use { provider -> - provider.refreshExpiredEntries(expiredEntries, testS3Client) - } + val testClient = TestS3Client(expectedCredentials) + + DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes).use { provider -> + val attributes = ExecutionContext.build { + this.attributes[S3Attributes.ExpressClient] = testClient + this.attributes[S3Attributes.Bucket] = "bucket" + } - val refreshedEntries = cache.entries - assertFalse(refreshedEntries.any { it.value.expiringCredentials.isExpired }) // none of the entries are expired - assertFalse(refreshedEntries.any { it.value.usedSinceLastRefresh }) // none of the entries have been used since the last refresh + // launch many async `resolve` calls, only one should call s3:CreateSession + val calls = (1..5).map { + async { provider.resolve(attributes) } + } + calls.awaitAll() + assertEquals(1, testClient.numCreateSession) + } } @Test - fun testRefreshExpiredEntriesHandlesFailures() = runTest { + fun testAsyncRefreshHandlesFailures() = runTest { val timeSource = TestTimeSource() val clock = ManualClock() + + // Entry expires in 30 seconds, refresh buffer is 1 minute. Next `resolve` call should trigger the async refresh val cache = S3ExpressCredentialsCache() + val successEntry = getCacheEntry(timeSource.markNow() + 30.seconds, bucket = "SuccessfulBucket", bootstrapCredentials = Credentials("1", "1", "1")) + val failedEntry = getCacheEntry(timeSource.markNow() + 30.seconds, bucket = "ExceptionBucket", bootstrapCredentials = Credentials("1", "1", "1")) + cache.put(successEntry.key, successEntry.value) + cache.put(failedEntry.key, failedEntry.value) val expectedCredentials = SessionCredentials { accessKeyId = "access" @@ -107,87 +130,72 @@ class DefaultS3ExpressCredentialsProviderTest { sessionToken = "session" expiration = clock.now() + 5.minutes } - val testS3Client = TestS3Client(expectedCredentials, throwExceptionOnBucket = "ExceptionBucket") - - val expiredEntries = listOf( - getEntry(timeSource.markNow() + 30.seconds, bucket = "ExceptionBucket", bootstrapCredentials = Credentials("1", "1", "1")), - getEntry(timeSource.markNow() + 1.minutes, bucket = "SuccessfulBucket", bootstrapCredentials = Credentials("2", "2", "2")), - getEntry(timeSource.markNow() + 1.minutes + 30.seconds, bucket = "SuccessfulBucket", bootstrapCredentials = Credentials("3", "3", "3")), - getEntry(timeSource.markNow() + 2.minutes, bucket = "ExceptionBucket", bootstrapCredentials = Credentials("4", "4", "4")), - getEntry(timeSource.markNow(), bucket = "SuccessfulBucket", bootstrapCredentials = Credentials("5", "5", "5")), - ) - expiredEntries.forEach { cache.put(it.key, it.value) } - assertEquals(5, cache.size) - - timeSource += 3.minutes - assertTrue(expiredEntries.all { it.value.expiringCredentials.isExpired }) // all entries are now expired - - DefaultS3ExpressCredentialsProvider(timeSource, clock, cache).use { provider -> - provider.refreshExpiredEntries(expiredEntries, testS3Client) + + // client will throw an exception when `ExceptionBucket` credentials are fetched, + // but there should be no crash + val testClient = TestS3Client(expectedCredentials, throwExceptionOnBucketNamed = "ExceptionBucket", baseCredentials = Credentials("1", "1", "1")) + + DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes).use { provider -> + val attributes = ExecutionContext.build { + this.attributes[S3Attributes.ExpressClient] = testClient + this.attributes[S3Attributes.Bucket] = "ExceptionBucket" + } + provider.resolve(attributes) + + attributes[S3Attributes.Bucket] = "SuccessfulBucket" + provider.resolve(attributes) } - val refreshedEntries = cache.entries - assertEquals(5, refreshedEntries.size) // no entries were removed - - // two entries failed to refresh, they are still expired. - assertEquals( - 2, - refreshedEntries - .filter { it.value.expiringCredentials.isExpired } - .size, - ) - assertTrue( - refreshedEntries - .filter { it.value.expiringCredentials.isExpired } - .all { it.key.bucket == "ExceptionBucket" }, - ) - - // three entries successfully refreshed and are no longer expired - assertEquals( - 3, - refreshedEntries - .filter { !it.value.expiringCredentials.isExpired } - .size, - ) - assertTrue( - refreshedEntries - .filter { !it.value.expiringCredentials.isExpired } - .all { it.key.bucket == "SuccessfulBucket" }, - ) + // close the provider, make sure all async refreshes are complete... + assertEquals(2, testClient.numCreateSession) } /** - * Get an instance of [Map.Entry] using the given [expiration], - * [bucket], and optional [bootstrapCredentials] and [sessionCredentials]. - */ - private fun getEntry( - expiration: ComparableTimeMark, - bucket: String = "bucket", - bootstrapCredentials: Credentials = Credentials(accessKeyId = "accessKeyId", secretAccessKey = "secretAccessKey", sessionToken = "sessionToken"), - sessionCredentials: Credentials = Credentials(accessKeyId = "s3AccessKeyId", secretAccessKey = "s3SecretAccessKey", sessionToken = "s3SessionToken"), - ): S3ExpressCredentialsCacheEntry = mapOf( - S3ExpressCredentialsCacheKey(bucket, bootstrapCredentials) to S3ExpressCredentialsCacheValue(ExpiringValue(sessionCredentials, expiration)), - ).entries.first() - - /** + * Get an instance of [Map.Entry] using the given [expiration], + * [bucket], and optional [bootstrapCredentials] and [sessionCredentials]. + */ + private fun getCacheEntry( + expiration: ComparableTimeMark, + bucket: String = "bucket", + bootstrapCredentials: Credentials = Credentials(accessKeyId = "accessKeyId", secretAccessKey = "secretAccessKey", sessionToken = "sessionToken"), + sessionCredentials: Credentials = Credentials(accessKeyId = "s3AccessKeyId", secretAccessKey = "s3SecretAccessKey", sessionToken = "s3SessionToken"), + ): S3ExpressCredentialsCacheEntry = mapOf( + S3ExpressCredentialsCacheKey(bucket, bootstrapCredentials) to S3ExpressCredentialsCacheValue(ExpiringValue(sessionCredentials, expiration)), + ).entries.first() + + + /** * A test S3Client used to mock calls to s3:CreateSession. * @param expectedCredentials the expected session credentials returned from s3:CreateSession * @param client the base S3 client used to implement other operations, though they are unused. - * @param throwExceptionOnBucket an optional bucket name, which when specified and present in the [CreateSessionRequest], will + * @param throwExceptionOnBucketNamed an optional bucket name, which when specified and present in the [CreateSessionRequest], will * cause the client to throw an exception instead of returning credentials. Used for testing s3:CreateSession failures. */ private class TestS3Client( - val expectedCredentials: SessionCredentials, - val client: S3Client = S3Client { }, - val throwExceptionOnBucket: String? = null, + val expectedCredentials: SessionCredentials, + val client: S3Client = S3Client { }, + val baseCredentials: Credentials? = null, + val throwExceptionOnBucketNamed: String? = null, ) : S3Client by client { + var numCreateSession = 0 + override val config: S3Client.Config + get() = baseCredentials?.let { + client.withConfig { + credentialsProvider = StaticCredentialsProvider(baseCredentials) + }.config + } ?: client.config + override suspend fun createSession(input: CreateSessionRequest): CreateSessionResponse { - throwExceptionOnBucket?.let { + println("in createSession for $input") + numCreateSession += 1 + println("numCreateSession: $numCreateSession") + + throwExceptionOnBucketNamed?.let { if (input.bucket == it) { - throw Exception("Failed to create session credentials for bucket: $throwExceptionOnBucket") + throw Exception("Failed to create session credentials for bucket: $throwExceptionOnBucketNamed") } } return CreateSessionResponse { credentials = expectedCredentials } } } -} + } From 71d20a58d2aafaa9e92fc07e24fdfc62a9c88987 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 14:51:06 -0500 Subject: [PATCH 112/145] ktlintFormat --- .../DefaultS3ExpressCredentialsProvider.kt | 1 - ...DefaultS3ExpressCredentialsProviderTest.kt | 39 +++++++++---------- .../express/S3ExpressCredentialsCacheTest.kt | 1 - 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 913bf93685c..1b376cbfa06 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -62,7 +62,6 @@ internal class DefaultS3ExpressCredentialsProvider( } catch (e: Exception) { logger.warn(e) { "Asynchronous refresh for ${key.bucket} failed." } } - } } } diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt index 86122ea8f4b..4d054645d48 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt @@ -17,8 +17,6 @@ import aws.smithy.kotlin.runtime.operation.ExecutionContext import aws.smithy.kotlin.runtime.time.ManualClock import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.delay -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import kotlin.test.* import kotlin.time.ComparableTimeMark @@ -151,20 +149,19 @@ class DefaultS3ExpressCredentialsProviderTest { } /** - * Get an instance of [Map.Entry] using the given [expiration], - * [bucket], and optional [bootstrapCredentials] and [sessionCredentials]. - */ - private fun getCacheEntry( - expiration: ComparableTimeMark, - bucket: String = "bucket", - bootstrapCredentials: Credentials = Credentials(accessKeyId = "accessKeyId", secretAccessKey = "secretAccessKey", sessionToken = "sessionToken"), - sessionCredentials: Credentials = Credentials(accessKeyId = "s3AccessKeyId", secretAccessKey = "s3SecretAccessKey", sessionToken = "s3SessionToken"), - ): S3ExpressCredentialsCacheEntry = mapOf( - S3ExpressCredentialsCacheKey(bucket, bootstrapCredentials) to S3ExpressCredentialsCacheValue(ExpiringValue(sessionCredentials, expiration)), - ).entries.first() - - - /** + * Get an instance of [Map.Entry] using the given [expiration], + * [bucket], and optional [bootstrapCredentials] and [sessionCredentials]. + */ + private fun getCacheEntry( + expiration: ComparableTimeMark, + bucket: String = "bucket", + bootstrapCredentials: Credentials = Credentials(accessKeyId = "accessKeyId", secretAccessKey = "secretAccessKey", sessionToken = "sessionToken"), + sessionCredentials: Credentials = Credentials(accessKeyId = "s3AccessKeyId", secretAccessKey = "s3SecretAccessKey", sessionToken = "s3SessionToken"), + ): S3ExpressCredentialsCacheEntry = mapOf( + S3ExpressCredentialsCacheKey(bucket, bootstrapCredentials) to S3ExpressCredentialsCacheValue(ExpiringValue(sessionCredentials, expiration)), + ).entries.first() + + /** * A test S3Client used to mock calls to s3:CreateSession. * @param expectedCredentials the expected session credentials returned from s3:CreateSession * @param client the base S3 client used to implement other operations, though they are unused. @@ -172,10 +169,10 @@ class DefaultS3ExpressCredentialsProviderTest { * cause the client to throw an exception instead of returning credentials. Used for testing s3:CreateSession failures. */ private class TestS3Client( - val expectedCredentials: SessionCredentials, - val client: S3Client = S3Client { }, - val baseCredentials: Credentials? = null, - val throwExceptionOnBucketNamed: String? = null, + val expectedCredentials: SessionCredentials, + val client: S3Client = S3Client { }, + val baseCredentials: Credentials? = null, + val throwExceptionOnBucketNamed: String? = null, ) : S3Client by client { var numCreateSession = 0 override val config: S3Client.Config @@ -198,4 +195,4 @@ class DefaultS3ExpressCredentialsProviderTest { return CreateSessionResponse { credentials = expectedCredentials } } } - } +} diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt index 0ca7608b97d..2387496056f 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCacheTest.kt @@ -71,6 +71,5 @@ public class S3ExpressCredentialsCacheTest { timeSource += 31.seconds assertTrue(expiringSessionCredentials.isExpiringWithin(30.seconds)) - } } From b78afec14292ccfdd8bc607ec044ec30b5532f9e Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 16:41:51 -0500 Subject: [PATCH 113/145] Remove `S3ExpressCredentialsProvider` interface --- .../s3/express/S3ExpressIntegration.kt | 11 ++- .../DefaultS3ExpressCredentialsProvider.kt | 15 ++-- .../express/S3ExpressCredentialsProvider.kt | 19 ----- ...DefaultS3ExpressCredentialsProviderTest.kt | 69 ++++++++++++++----- 4 files changed, 68 insertions(+), 46 deletions(-) delete mode 100644 services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 1b399adc328..f95942dc742 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -42,18 +42,17 @@ class S3ExpressIntegration : KotlinIntegration { val ExpressCredentialsProvider: ConfigProperty = ConfigProperty { name = "expressCredentialsProvider" - symbol = buildSymbol { - name = "S3ExpressCredentialsProvider" - nullable = false - namespace = "aws.sdk.kotlin.services.s3.express" - } + symbol = RuntimeTypes.Auth.Credentials.AwsCredentials.CredentialsProvider documentation = """ Credentials provider to be used for making requests to S3 Express. """.trimIndent() propertyType = ConfigPropertyType.Custom( render = { _, writer -> - writer.write("public val #1L: #2T = builder.#1L ?: #2T.default()", name, symbol) + writer.write("public val #1L: #2T = builder.#1L ?: #3T()", name, symbol, buildSymbol { + name = "DefaultS3ExpressCredentialsProvider" + namespace = "aws.sdk.kotlin.services.s3.express" + }) }, renderBuilder = { prop, writer -> prop.documentation?.let(writer::dokka) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 1b376cbfa06..5566535b4a6 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -23,13 +23,13 @@ import kotlin.time.Duration.Companion.minutes import kotlin.time.TimeSource /** - * The default implementation of [S3ExpressCredentialsProvider]. Performs best-effort asynchronous refresh if the - * cached credentials are within their refresh window ([REFRESH_BUFFER] away from expiration) during a call to [resolve]. + * The default implementation of a credentials provider for S3 Express One Zone. Performs best-effort asynchronous refresh + * if the cached credentials are within their refresh window ([REFRESH_BUFFER] away from expiration) during a call to [resolve]. * Otherwise, performs synchronous refresh. * * @param timeSource the time source to use. defaults to [TimeSource.Monotonic] * @param clock the clock to use. defaults to [Clock.System]. note: the clock is only used to get an initial [Duration] - * until credentials expiration. + * until credentials expiration, [timeSource] is used as the source of truth for credentials expiration. * @param credentialsCache an [S3ExpressCredentialsCache] to be used for caching session credentials, defaults to * [S3ExpressCredentialsCache]. * @param refreshBuffer an optional [Duration] representing the duration before expiration that [Credentials] @@ -40,9 +40,12 @@ internal class DefaultS3ExpressCredentialsProvider( private val clock: Clock = Clock.System, private val credentialsCache: S3ExpressCredentialsCache = S3ExpressCredentialsCache(), private val refreshBuffer: Duration = 1.minutes, -) : S3ExpressCredentialsProvider, CloseableCredentialsProvider, SdkManagedBase(), CoroutineScope { +) : CloseableCredentialsProvider, SdkManagedBase(), CoroutineScope { + public constructor(): this(TimeSource.Monotonic) + private lateinit var client: S3Client override val coroutineContext: CoroutineContext = Job() + CoroutineName("DefaultS3ExpressCredentialsProvider") + val jobs = mutableListOf() override suspend fun resolve(attributes: Attributes): Credentials { client = attributes[S3Attributes.ExpressClient] as S3Client @@ -73,7 +76,9 @@ internal class DefaultS3ExpressCredentialsProvider( }.value } - override fun close() = coroutineContext.cancel(null) + override fun close() { + coroutineContext.cancel(null) + } internal suspend fun createSessionCredentials(bucket: String, client: S3Client): ExpiringValue = client.createSession { this.bucket = bucket }.credentials!!.let { diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt deleted file mode 100644 index 3b32ff9fc92..00000000000 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsProvider.kt +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package aws.sdk.kotlin.services.s3.express - -import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider - -/** - * A credentials provider used for making requests to S3 Express One Zone directory buckets. - */ -public interface S3ExpressCredentialsProvider : CredentialsProvider { - public companion object { - /** - * Create an instance of the default [S3ExpressCredentialsProvider] implementation - */ - public fun default(): S3ExpressCredentialsProvider = DefaultS3ExpressCredentialsProvider() - } -} diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt index 4d054645d48..36dcc261601 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt @@ -10,13 +10,11 @@ import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.CreateSessionRequest import aws.sdk.kotlin.services.s3.model.CreateSessionResponse import aws.sdk.kotlin.services.s3.model.SessionCredentials -import aws.sdk.kotlin.services.s3.withConfig import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.io.use import aws.smithy.kotlin.runtime.operation.ExecutionContext import aws.smithy.kotlin.runtime.time.ManualClock -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.* import kotlinx.coroutines.test.runTest import kotlin.test.* import kotlin.time.ComparableTimeMark @@ -25,6 +23,8 @@ import kotlin.time.Duration.Companion.seconds import kotlin.time.TestTimeSource class DefaultS3ExpressCredentialsProviderTest { + private val DEFAULT_BASE_CREDENTIALS = Credentials("accessKeyId", "secretAccessKey", "sessionToken") + @Test fun testCreateSessionCredentials() = runTest { val timeSource = TestTimeSource() @@ -72,8 +72,10 @@ class DefaultS3ExpressCredentialsProviderTest { } provider.resolve(attributes) - assertEquals(1, testClient.numCreateSession) } + // close the provider, make sure all async refreshes are complete... + delay(1.seconds) + assertEquals(1, testClient.numCreateSession) } @Test @@ -101,13 +103,15 @@ class DefaultS3ExpressCredentialsProviderTest { this.attributes[S3Attributes.Bucket] = "bucket" } - // launch many async `resolve` calls, only one should call s3:CreateSession val calls = (1..5).map { async { provider.resolve(attributes) } } calls.awaitAll() - assertEquals(1, testClient.numCreateSession) } + + // close the provider, make sure all async refreshes are complete... + delay(5.seconds) + assertEquals(1, testClient.numCreateSession) } @Test @@ -143,11 +147,50 @@ class DefaultS3ExpressCredentialsProviderTest { attributes[S3Attributes.Bucket] = "SuccessfulBucket" provider.resolve(attributes) } - // close the provider, make sure all async refreshes are complete... + delay(1.seconds) assertEquals(2, testClient.numCreateSession) } + @Test + fun testAsyncRefreshClosesImmediately() = runTest { + val timeSource = TestTimeSource() + val clock = ManualClock() + + // Entry expires in 30 seconds, refresh buffer is 1 minute. Next `resolve` call should trigger the async refresh + val cache = S3ExpressCredentialsCache() + val entry = getCacheEntry(timeSource.markNow() + 30.seconds) + cache.put(entry.key, entry.value) + + val expectedCredentials = SessionCredentials { + accessKeyId = "access" + secretAccessKey = "secret" + sessionToken = "session" + expiration = clock.now() + 5.minutes + } + + val provider = DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes) + + val blockingTestS3Client = object : TestS3Client(expectedCredentials) { + override suspend fun createSession(input: CreateSessionRequest): CreateSessionResponse { + delay(10.seconds) + numCreateSession += 1 + return CreateSessionResponse { credentials = expectedCredentials } + } + } + + val attributes = ExecutionContext.build { + this.attributes[S3Attributes.ExpressClient] = blockingTestS3Client + this.attributes[S3Attributes.Bucket] = "bucket" + } + + withTimeout(5.seconds) { + provider.resolve(attributes) + provider.close() + } + assertEquals(0, blockingTestS3Client.numCreateSession) + } + /** * Get an instance of [Map.Entry] using the given [expiration], * [bucket], and optional [bootstrapCredentials] and [sessionCredentials]. @@ -168,19 +211,13 @@ class DefaultS3ExpressCredentialsProviderTest { * @param throwExceptionOnBucketNamed an optional bucket name, which when specified and present in the [CreateSessionRequest], will * cause the client to throw an exception instead of returning credentials. Used for testing s3:CreateSession failures. */ - private class TestS3Client( + private open inner class TestS3Client( val expectedCredentials: SessionCredentials, - val client: S3Client = S3Client { }, - val baseCredentials: Credentials? = null, + val baseCredentials: Credentials = DEFAULT_BASE_CREDENTIALS, + val client: S3Client = S3Client { credentialsProvider = StaticCredentialsProvider(baseCredentials) }, val throwExceptionOnBucketNamed: String? = null, ) : S3Client by client { var numCreateSession = 0 - override val config: S3Client.Config - get() = baseCredentials?.let { - client.withConfig { - credentialsProvider = StaticCredentialsProvider(baseCredentials) - }.config - } ?: client.config override suspend fun createSession(input: CreateSessionRequest): CreateSessionResponse { println("in createSession for $input") From b5557b419ccb255731b049715697c3a441052975 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 16:44:05 -0500 Subject: [PATCH 114/145] Update single flight group use --- .../DefaultS3ExpressCredentialsProvider.kt | 17 +++++++---------- .../s3/express/S3ExpressCredentialsCache.kt | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 5566535b4a6..0ae9ad751ee 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -45,7 +45,6 @@ internal class DefaultS3ExpressCredentialsProvider( private lateinit var client: S3Client override val coroutineContext: CoroutineContext = Job() + CoroutineName("DefaultS3ExpressCredentialsProvider") - val jobs = mutableListOf() override suspend fun resolve(attributes: Attributes): Credentials { client = attributes[S3Attributes.ExpressClient] as S3Client @@ -56,15 +55,13 @@ internal class DefaultS3ExpressCredentialsProvider( ?.takeIf { !it.expiringCredentials.isExpired } ?.also { if (it.expiringCredentials.isExpiringWithin(refreshBuffer)) { - it.sfg.singleFlight { - logger.trace { "Credentials for ${key.bucket} are within their refresh window, performing asynchronous refresh..." } - launch(coroutineContext) { - try { - val sessionCredentials = createSessionCredentials(key.bucket, client) - credentialsCache.put(key, S3ExpressCredentialsCacheValue(sessionCredentials)) - } catch (e: Exception) { - logger.warn(e) { "Asynchronous refresh for ${key.bucket} failed." } - } + logger.trace { "Credentials for ${key.bucket} are within their refresh window, performing asynchronous refresh..." } + launch(coroutineContext) { + try { + val sessionCredentials = it.sfg.singleFlight { createSessionCredentials(key.bucket, client) } + credentialsCache.put(key, S3ExpressCredentialsCacheValue(sessionCredentials)) + } catch (e: Exception) { + logger.warn(e) { "Asynchronous refresh for ${key.bucket} failed." } } } } diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index 1c1b2616e92..951d2ecb6fe 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -35,7 +35,7 @@ internal data class S3ExpressCredentialsCacheValue( /** * A [SingleFlightGroup] used to de-duplicate asynchronous refresh attempts */ - val sfg: SingleFlightGroup = SingleFlightGroup(), + val sfg: SingleFlightGroup> = SingleFlightGroup(), ) /** From 8f747d7523a3d629a9507d2984b13147b1710e4a Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 16:44:28 -0500 Subject: [PATCH 115/145] ktlint --- .../s3/express/S3ExpressIntegration.kt | 13 +++++++++---- .../express/DefaultS3ExpressCredentialsProvider.kt | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index f95942dc742..96c5c4f9bf4 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -49,10 +49,15 @@ class S3ExpressIntegration : KotlinIntegration { propertyType = ConfigPropertyType.Custom( render = { _, writer -> - writer.write("public val #1L: #2T = builder.#1L ?: #3T()", name, symbol, buildSymbol { - name = "DefaultS3ExpressCredentialsProvider" - namespace = "aws.sdk.kotlin.services.s3.express" - }) + writer.write( + "public val #1L: #2T = builder.#1L ?: #3T()", + name, + symbol, + buildSymbol { + name = "DefaultS3ExpressCredentialsProvider" + namespace = "aws.sdk.kotlin.services.s3.express" + }, + ) }, renderBuilder = { prop, writer -> prop.documentation?.let(writer::dokka) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 0ae9ad751ee..44fd6e9b417 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -41,7 +41,7 @@ internal class DefaultS3ExpressCredentialsProvider( private val credentialsCache: S3ExpressCredentialsCache = S3ExpressCredentialsCache(), private val refreshBuffer: Duration = 1.minutes, ) : CloseableCredentialsProvider, SdkManagedBase(), CoroutineScope { - public constructor(): this(TimeSource.Monotonic) + public constructor() : this(TimeSource.Monotonic) private lateinit var client: S3Client override val coroutineContext: CoroutineContext = Job() + CoroutineName("DefaultS3ExpressCredentialsProvider") From 2d15cc9e07908ab496b918bcf2030323982fd2cb Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 21 Feb 2024 17:01:36 -0500 Subject: [PATCH 116/145] update docs --- .../services/s3/express/DefaultS3ExpressCredentialsProvider.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 44fd6e9b417..4004b2dd698 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -24,7 +24,7 @@ import kotlin.time.TimeSource /** * The default implementation of a credentials provider for S3 Express One Zone. Performs best-effort asynchronous refresh - * if the cached credentials are within their refresh window ([REFRESH_BUFFER] away from expiration) during a call to [resolve]. + * if the cached credentials are expiring within a [refreshBuffer] during a call to [resolve]. * Otherwise, performs synchronous refresh. * * @param timeSource the time source to use. defaults to [TimeSource.Monotonic] From 60e9f6d19dfb6127f2e76fdf2d67650cbed62eff Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 22 Feb 2024 09:26:58 -0500 Subject: [PATCH 117/145] add runBlocking to delay calls --- ...DefaultS3ExpressCredentialsProviderTest.kt | 45 +++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt index 36dcc261601..1ec1e51ad4e 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt @@ -18,6 +18,7 @@ import kotlinx.coroutines.* import kotlinx.coroutines.test.runTest import kotlin.test.* import kotlin.time.ComparableTimeMark +import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds import kotlin.time.TestTimeSource @@ -46,6 +47,36 @@ class DefaultS3ExpressCredentialsProviderTest { } } + @Test + fun testSyncRefresh() = runTest { + val timeSource = TestTimeSource() + val clock = ManualClock() + + // Entry expired 30 seconds ago, next `resolve` call should trigger a sync refresh + val cache = S3ExpressCredentialsCache() + val entry = getCacheEntry(timeSource.markNow() - 30.seconds) + cache.put(entry.key, entry.value) + + val expectedCredentials = SessionCredentials { + accessKeyId = "access" + secretAccessKey = "secret" + sessionToken = "session" + expiration = clock.now() + 5.minutes + } + + val testClient = TestS3Client(expectedCredentials) + DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes).use { provider -> + val attributes = ExecutionContext.build { + this.attributes[S3Attributes.ExpressClient] = testClient + this.attributes[S3Attributes.Bucket] = "bucket" + } + + provider.resolve(attributes) + } + assertEquals(1, testClient.numCreateSession) + } + + @Test fun testAsyncRefresh() = runTest { val timeSource = TestTimeSource() @@ -64,17 +95,16 @@ class DefaultS3ExpressCredentialsProviderTest { } val testClient = TestS3Client(expectedCredentials) - DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes).use { provider -> val attributes = ExecutionContext.build { this.attributes[S3Attributes.ExpressClient] = testClient this.attributes[S3Attributes.Bucket] = "bucket" } - provider.resolve(attributes) } + // close the provider, make sure all async refreshes are complete... - delay(1.seconds) + runBlocking { delay(10.milliseconds) } assertEquals(1, testClient.numCreateSession) } @@ -110,7 +140,7 @@ class DefaultS3ExpressCredentialsProviderTest { } // close the provider, make sure all async refreshes are complete... - delay(5.seconds) + runBlocking { delay(10.milliseconds) } assertEquals(1, testClient.numCreateSession) } @@ -147,8 +177,9 @@ class DefaultS3ExpressCredentialsProviderTest { attributes[S3Attributes.Bucket] = "SuccessfulBucket" provider.resolve(attributes) } + // close the provider, make sure all async refreshes are complete... - delay(1.seconds) + runBlocking { delay(10.milliseconds) } assertEquals(2, testClient.numCreateSession) } @@ -220,10 +251,8 @@ class DefaultS3ExpressCredentialsProviderTest { var numCreateSession = 0 override suspend fun createSession(input: CreateSessionRequest): CreateSessionResponse { - println("in createSession for $input") numCreateSession += 1 - println("numCreateSession: $numCreateSession") - + println("i numCreateSession $numCreateSession") throwExceptionOnBucketNamed?.let { if (input.bucket == it) { throw Exception("Failed to create session credentials for bucket: $throwExceptionOnBucketNamed") From 036f7ee0665e2a200d5497ed629add82ff7b4e91 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 22 Feb 2024 10:28:02 -0500 Subject: [PATCH 118/145] CI From d7b176db6ae0d43ffb5687ad1b2d067efe440f36 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 22 Feb 2024 10:37:25 -0500 Subject: [PATCH 119/145] ktlint --- .../s3/express/DefaultS3ExpressCredentialsProviderTest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt index 1ec1e51ad4e..bdb43512ae9 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt @@ -76,7 +76,6 @@ class DefaultS3ExpressCredentialsProviderTest { assertEquals(1, testClient.numCreateSession) } - @Test fun testAsyncRefresh() = runTest { val timeSource = TestTimeSource() From 585a85fb2450fe518ed950ae72ede6b8027f1885 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 22 Feb 2024 15:54:15 -0500 Subject: [PATCH 120/145] Set name on client instead of operation benchmark --- .../benchmarks/service/definitions/S3ExpressBenchmark.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt index f87f3684b42..f1bf277cff0 100644 --- a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt +++ b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt @@ -25,6 +25,7 @@ class S3ExpressBenchmark : ServiceBenchmark { @OptIn(ExperimentalApi::class) override suspend fun client() = S3Client.fromEnvironment { + clientName = "S3Express" retryStrategy = Common.noRetries telemetryProvider = Common.telemetryProvider httpClient { @@ -71,7 +72,7 @@ class S3ExpressBenchmark : ServiceBenchmark { } } - private val getObjectBenchmark = object : AbstractOperationBenchmark("s3express:GetObject") { + private val getObjectBenchmark = object : AbstractOperationBenchmark("GetObject") { override suspend fun setup(client: S3Client) { client.putObject { bucket = bucketName From a4341535acfff52d30c5b70ea82ac75ed3f42a9d Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 22 Feb 2024 15:57:32 -0500 Subject: [PATCH 121/145] Remove unnecessary function --- services/s3/e2eTest/src/S3TestUtils.kt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 4b61a1e5bc6..143c399dba6 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -29,9 +29,7 @@ object S3TestUtils { suspend fun getTestBucket(client: S3Client): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX) - suspend fun getTestDirectoryBucket(client: S3Client, suffix: String) = getDirectoryBucket(client, suffix) - - private suspend fun getDirectoryBucket(client: S3Client, suffix: String): String = withTimeout(60.seconds) { + suspend fun getTestDirectoryBucket(client: S3Client, suffix: String) = withTimeout(60.seconds) { var testBucket = client.listBuckets() .buckets ?.mapNotNull { it.name } @@ -43,7 +41,7 @@ object S3TestUtils { UUID.randomUUID().toString().subSequence(0 until (S3_MAX_BUCKET_NAME_LENGTH - TEST_BUCKET_PREFIX.length - suffix.ensurePrefix("--").length)) + suffix.ensurePrefix("--") - println("Creating S3 bucket: $testBucket") + println("Creating S3 Express directory bucket: $testBucket") val availabilityZone = testBucket // s3-test-bucket-UUID--use1-az4--x-s3 .removeSuffix(S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX) // s3-test-bucket-UUID--use1-az4 @@ -63,7 +61,6 @@ object S3TestUtils { } } } - testBucket } From 7454ca598d9630cb6fcccb0ac019bf6c203b7ce4 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 22 Feb 2024 16:00:16 -0500 Subject: [PATCH 122/145] `createSessionCredentials` also insert the credentials into the cache --- .../DefaultS3ExpressCredentialsProvider.kt | 16 +++++++++------- .../DefaultS3ExpressCredentialsProviderTest.kt | 5 ++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index 4004b2dd698..db45b67f57c 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -58,8 +58,7 @@ internal class DefaultS3ExpressCredentialsProvider( logger.trace { "Credentials for ${key.bucket} are within their refresh window, performing asynchronous refresh..." } launch(coroutineContext) { try { - val sessionCredentials = it.sfg.singleFlight { createSessionCredentials(key.bucket, client) } - credentialsCache.put(key, S3ExpressCredentialsCacheValue(sessionCredentials)) + it.sfg.singleFlight { createSessionCredentials(key, client) } } catch (e: Exception) { logger.warn(e) { "Asynchronous refresh for ${key.bucket} failed." } } @@ -68,21 +67,24 @@ internal class DefaultS3ExpressCredentialsProvider( } ?.expiringCredentials ?.value - ?: createSessionCredentials(key.bucket, client).also { - credentialsCache.put(key, S3ExpressCredentialsCacheValue(it)) - }.value + ?: createSessionCredentials(key, client).value } override fun close() { coroutineContext.cancel(null) } - internal suspend fun createSessionCredentials(bucket: String, client: S3Client): ExpiringValue = - client.createSession { this.bucket = bucket }.credentials!!.let { + /** + * Create a new set of session credentials by calling s3:CreateSession and then store them in the cache. + */ + internal suspend fun createSessionCredentials(key: S3ExpressCredentialsCacheKey, client: S3Client): ExpiringValue = + client.createSession { bucket = key.bucket }.credentials!!.let { ExpiringValue( Credentials(it.accessKeyId, it.secretAccessKey, it.sessionToken, it.expiration), expiresAt = timeSource.markNow() + clock.now().until(it.expiration), ) + }.also { + credentialsCache.put(key, S3ExpressCredentialsCacheValue(it)) } @OptIn(ExperimentalApi::class) diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt index bdb43512ae9..1b2ee9ee7c6 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt @@ -41,7 +41,10 @@ class DefaultS3ExpressCredentialsProviderTest { val client = TestS3Client(expectedCredentials) DefaultS3ExpressCredentialsProvider(timeSource, clock).use { provider -> - val credentials = provider.createSessionCredentials("bucket", client) + val credentials = provider.createSessionCredentials( + S3ExpressCredentialsCacheKey("bucket", DEFAULT_BASE_CREDENTIALS), + client + ) assertFalse(credentials.isExpired) assertEquals(timeSource.markNow() + 5.minutes, credentials.expiresAt) } From ef388257cd7cb2965538a0ac0a7ab8695c186093 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 22 Feb 2024 16:12:29 -0500 Subject: [PATCH 123/145] ktlint --- .../s3/express/DefaultS3ExpressCredentialsProviderTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt index 1b2ee9ee7c6..f0fa7f6ba91 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt @@ -43,7 +43,7 @@ class DefaultS3ExpressCredentialsProviderTest { DefaultS3ExpressCredentialsProvider(timeSource, clock).use { provider -> val credentials = provider.createSessionCredentials( S3ExpressCredentialsCacheKey("bucket", DEFAULT_BASE_CREDENTIALS), - client + client, ) assertFalse(credentials.isExpired) assertEquals(timeSource.markNow() + 5.minutes, credentials.expiresAt) From 000bb4dd2786e7845f8b53a497c6cb1c3a41cf44 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 22 Feb 2024 17:08:18 -0500 Subject: [PATCH 124/145] also remove prefix on PutObject --- .../kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt index f1bf277cff0..73df450e939 100644 --- a/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt +++ b/tests/benchmarks/service-benchmarks/jvm/src/aws/sdk/kotlin/benchmarks/service/definitions/S3ExpressBenchmark.kt @@ -55,7 +55,7 @@ class S3ExpressBenchmark : ServiceBenchmark { client.deleteBucket { bucket = bucketName } } - private val putObjectBenchmark = object : AbstractOperationBenchmark("s3express:PutObject") { + private val putObjectBenchmark = object : AbstractOperationBenchmark("PutObject") { override suspend fun transact(client: S3Client) { client.putObject { bucket = bucketName From f5cdfcad516ca14dcd8d58354bf990c148c37192 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 22 Feb 2024 17:15:43 -0500 Subject: [PATCH 125/145] ktlint --- services/s3/e2eTest/src/S3TestUtils.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index d0f505837e7..396f2a67892 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -107,8 +107,8 @@ object S3TestUtils { if (testBucket == null) { // Adding S3 Express suffix surpasses the bucket name length limit... trim the UUID if needed testBucket = TEST_BUCKET_PREFIX + - UUID.randomUUID().toString().subSequence(0 until (S3_MAX_BUCKET_NAME_LENGTH - TEST_BUCKET_PREFIX.length - suffix.ensurePrefix("--").length)) + - suffix.ensurePrefix("--") + UUID.randomUUID().toString().subSequence(0 until (S3_MAX_BUCKET_NAME_LENGTH - TEST_BUCKET_PREFIX.length - suffix.ensurePrefix("--").length)) + + suffix.ensurePrefix("--") println("Creating S3 Express directory bucket: $testBucket") From 0bdd2e4a74d71232e55c091b7ba85210f869c6df Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Thu, 22 Feb 2024 17:23:36 -0500 Subject: [PATCH 126/145] only print new statuses --- services/s3/e2eTest/src/S3TestUtils.kt | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 396f2a67892..51bc9ff9684 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -273,21 +273,25 @@ object S3TestUtils { operation: String, ) { withTimeout(timeoutAfter) { + var status: String? = null while (true) { - val status = s3ControlClient.describeMultiRegionAccessPointOperation { + val latestStatus = s3ControlClient.describeMultiRegionAccessPointOperation { accountId = testAccountId requestTokenArn = request }.asyncOperation?.requestStatus - println("Waiting on $operation operation. Status: $status ") - - if (status == "SUCCEEDED") { - println("$operation operation succeeded.") - return@withTimeout + when (latestStatus) { + "SUCCEEDED" -> { + println("$operation operation succeeded.") + return@withTimeout + } + "FAILED" -> throw IllegalStateException("$operation operation failed") + else -> { if (status == null || latestStatus != status) { + println("Waiting on $operation operation. Status: $latestStatus ") + status = latestStatus + }} } - check(status != "FAILED") { "$operation operation failed" } - delay(10.seconds) // Avoid constant status checks } } From b989db68c5aea8268874acb2db41dfa530dbb9fe Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Mon, 26 Feb 2024 16:50:57 -0500 Subject: [PATCH 127/145] @InternalSdkApi --- .../s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt index 053b121b5ac..13ce1311198 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/S3Attributes.kt @@ -4,12 +4,14 @@ */ package aws.sdk.kotlin.services.s3 +import aws.sdk.kotlin.runtime.InternalSdkApi import aws.smithy.kotlin.runtime.client.SdkClient import aws.smithy.kotlin.runtime.collections.AttributeKey /** * Execution context attributes specific to S3 */ +@InternalSdkApi public object S3Attributes { /** * The name of the bucket requests are being made to From 890b12e96509ce5674935e9dd4ea34246cc09470 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Mon, 26 Feb 2024 16:56:31 -0500 Subject: [PATCH 128/145] Add expiration to log --- .../services/s3/express/DefaultS3ExpressCredentialsProvider.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index db45b67f57c..bdfd2299adf 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -55,7 +55,7 @@ internal class DefaultS3ExpressCredentialsProvider( ?.takeIf { !it.expiringCredentials.isExpired } ?.also { if (it.expiringCredentials.isExpiringWithin(refreshBuffer)) { - logger.trace { "Credentials for ${key.bucket} are within their refresh window, performing asynchronous refresh..." } + logger.trace { "Credentials for ${key.bucket} are expiring in ${it.expiringCredentials.expiresAt} and are within their refresh window, performing asynchronous refresh..." } launch(coroutineContext) { try { it.sfg.singleFlight { createSessionCredentials(key, client) } From 4258ab7b5f6275c7caa4cae8b9bd333de2672225 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Mon, 26 Feb 2024 17:08:09 -0500 Subject: [PATCH 129/145] rename to baseCredentials --- .../sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt index 951d2ecb6fe..054322c57a3 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCredentialsCache.kt @@ -24,7 +24,7 @@ internal data class S3ExpressCredentialsCacheKey( /** * The base credentials used to resolve session credentials */ - val credentials: Credentials, + val baseCredentials: Credentials, ) internal data class S3ExpressCredentialsCacheValue( From f5b99f445f7778fb80964a4dd5e3276e6768f039 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Mon, 26 Feb 2024 18:07:07 -0500 Subject: [PATCH 130/145] Handle input changing through interceptors --- .../flexiblechecksums/FlexibleChecksumsRequest.kt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 5fb69ebab68..85c5960f956 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -52,8 +52,12 @@ class FlexibleChecksumsRequest : KotlinIntegration { .members() .first { it.memberName == httpChecksumTrait.requestAlgorithmMember.get() } - writer.write("op.interceptors.add(#T<#T>())", interceptorSymbol, inputSymbol) - writer.withBlock("input.${requestAlgorithmMember.defaultName()}?.let {", "}") { + val requestAlgorithmMemberName = ctx.symbolProvider.toMemberName(requestAlgorithmMember) + + writer.withBlock("op.interceptors.add(#T<#T>() {", "})", interceptorSymbol, inputSymbol) { + writer.write("input.#L?.value", requestAlgorithmMemberName) + } + writer.withBlock("input.#L?.let {", "}", requestAlgorithmMemberName) { writer.write("op.context[#T.ChecksumAlgorithm] = it.value", RuntimeTypes.HttpClient.Operation.HttpOperationContext) } } From 4b505c49eac746c6b3e1b0f86d56f8a3ca710419 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Mon, 26 Feb 2024 18:13:26 -0500 Subject: [PATCH 131/145] ktlint --- .../customization/flexiblechecksums/FlexibleChecksumsRequest.kt | 1 - services/s3/e2eTest/src/S3TestUtils.kt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt index 85c5960f956..8799ac5364e 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/flexiblechecksums/FlexibleChecksumsRequest.kt @@ -8,7 +8,6 @@ import software.amazon.smithy.aws.traits.HttpChecksumTrait import software.amazon.smithy.kotlin.codegen.KotlinSettings import software.amazon.smithy.kotlin.codegen.core.KotlinWriter import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes -import software.amazon.smithy.kotlin.codegen.core.defaultName import software.amazon.smithy.kotlin.codegen.core.withBlock import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration import software.amazon.smithy.kotlin.codegen.model.* diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 51bc9ff9684..c709fffe01f 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -289,7 +289,7 @@ object S3TestUtils { else -> { if (status == null || latestStatus != status) { println("Waiting on $operation operation. Status: $latestStatus ") status = latestStatus - }} + } } } delay(10.seconds) // Avoid constant status checks From cefbf91dd84c6e0bf722571346f6aca9a21dde5b Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 27 Feb 2024 09:23:25 -0500 Subject: [PATCH 132/145] Delegate all enablement logic to `isEnabledFor` --- .../customization/s3/express/S3ExpressIntegration.kt | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 96c5c4f9bf4..058fa932a9e 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -140,8 +140,8 @@ class S3ExpressIntegration : KotlinIntegration { override val order: Byte = -1 // Render before flexible checksums - override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = - !op.isS3UploadPart && (op.hasTrait() || op.hasTrait()) + override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean = !op.isS3UploadPart && + (op.hasTrait() || (op.hasTrait() && op.expectTrait().isRequestChecksumRequired)) override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) { val interceptorSymbol = buildSymbol { @@ -158,12 +158,7 @@ class S3ExpressIntegration : KotlinIntegration { // S3 models a header name x-amz-sdk-checksum-algorithm representing the name of the checksum algorithm used val checksumHeaderName = checksumAlgorithmMember?.getTrait()?.value - val checksumRequired = - op.hasTrait() || httpChecksumTrait?.isRequestChecksumRequired == true - - if (checksumRequired) { - writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq() ?: ""}))", interceptorSymbol) - } + writer.write("op.interceptors.add(#T(${checksumHeaderName?.dq() ?: ""}))", interceptorSymbol) } } From f92950cbf191283d2862f45cca41ec4aff98fc86 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 27 Feb 2024 10:50:45 -0500 Subject: [PATCH 133/145] Prevent generating empty `SigningContextAttributeKey` list --- .../SigV4S3ExpressAuthSchemeIntegration.kt | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt index 4ec6ca4986c..17d249bc063 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/SigV4S3ExpressAuthSchemeIntegration.kt @@ -65,7 +65,7 @@ internal val SigV4S3ExpressAuthSchemeSymbol = buildSymbol { private object SigV4S3ExpressEndpointCustomization : EndpointCustomization { override val propertyRenderers: Map = mapOf( - "authSchemes" to ::renderAuthSchemes, + "authSchemes" to ::renderAuthScheme, ) } @@ -103,26 +103,26 @@ class SigV4S3ExpressAuthSchemeHandler : AuthSchemeHandler { } } -private fun renderAuthSchemes(writer: KotlinWriter, authSchemes: Expression, expressionRenderer: ExpressionRenderer) { - writer.writeInline("#T to ", RuntimeTypes.SmithyClient.Endpoints.SigningContextAttributeKey) - writer.withBlock("listOf(", ")") { - authSchemes.toNode().expectArrayNode().forEach { - val scheme = it.expectObjectNode() - val schemeName = scheme.expectStringMember("name").value - val authFactoryFn = if (schemeName == "sigv4-s3express") sigV4S3ExpressSymbol else return@forEach +private fun renderAuthScheme(writer: KotlinWriter, authSchemes: Expression, expressionRenderer: ExpressionRenderer) { + val expressScheme = authSchemes.toNode().expectArrayNode().find { + it.expectObjectNode().expectStringMember("name").value == "sigv4-s3express" + }?.expectObjectNode() - withBlock("#T(", "),", authFactoryFn) { + expressScheme?.let { + writer.writeInline("#T to ", RuntimeTypes.SmithyClient.Endpoints.SigningContextAttributeKey) + writer.withBlock("listOf(", ")") { + withBlock("#T(", "),", sigV4S3ExpressSymbol) { // we delegate back to the expression visitor for each of these fields because it's possible to // encounter template strings throughout writeInline("serviceName = ") - renderOrElse(expressionRenderer, scheme.getStringMember("signingName"), "null") + renderOrElse(expressionRenderer, expressScheme.getStringMember("signingName"), "null") writeInline("disableDoubleUriEncode = ") - renderOrElse(expressionRenderer, scheme.getBooleanMember("disableDoubleEncoding"), "false") + renderOrElse(expressionRenderer, expressScheme.getBooleanMember("disableDoubleEncoding"), "false") writeInline("signingRegion = ") - renderOrElse(expressionRenderer, scheme.getStringMember("signingRegion"), "null") + renderOrElse(expressionRenderer, expressScheme.getStringMember("signingRegion"), "null") } } } From d04cad942b094bc746b16b52758df2246e13fe5b Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 27 Feb 2024 10:51:02 -0500 Subject: [PATCH 134/145] Add S3 Express to the end of the list --- .../codegen/customization/s3/express/S3ExpressIntegration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 058fa932a9e..586b17bf28f 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -88,7 +88,7 @@ class S3ExpressIntegration : KotlinIntegration { serviceShapeBuilder.addTrait(SigV4S3ExpressAuthTrait()) - val authTrait = AuthTrait(mutableSetOf(SigV4S3ExpressAuthTrait.ID) + serviceShape.expectTrait(AuthTrait::class.java).valueSet) + val authTrait = AuthTrait(serviceShape.expectTrait(AuthTrait::class.java).valueSet + mutableSetOf(SigV4S3ExpressAuthTrait.ID)) serviceShapeBuilder.addTrait(authTrait) // Add the new shape and update the service shape's AuthTrait From e302fc709e0bab15f98d0df6615712f2425ce8d1 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Tue, 27 Feb 2024 10:57:35 -0500 Subject: [PATCH 135/145] Update tests to handle async refresh initiation --- ...DefaultS3ExpressCredentialsProviderTest.kt | 69 +++++++++++-------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt index f0fa7f6ba91..861c77ea149 100644 --- a/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt +++ b/services/s3/common/test/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProviderTest.kt @@ -97,16 +97,22 @@ class DefaultS3ExpressCredentialsProviderTest { } val testClient = TestS3Client(expectedCredentials) - DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes).use { provider -> - val attributes = ExecutionContext.build { - this.attributes[S3Attributes.ExpressClient] = testClient - this.attributes[S3Attributes.Bucket] = "bucket" - } - provider.resolve(attributes) + + val provider = DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes) + + val attributes = ExecutionContext.build { + this.attributes[S3Attributes.ExpressClient] = testClient + this.attributes[S3Attributes.Bucket] = "bucket" } + provider.resolve(attributes) + + // allow the async refresh to initiate before closing the provider + runBlocking { delay(50.milliseconds) } // close the provider, make sure all async refreshes are complete... - runBlocking { delay(10.milliseconds) } + provider.close() + runBlocking { delay(50.milliseconds) } + assertEquals(1, testClient.numCreateSession) } @@ -129,20 +135,24 @@ class DefaultS3ExpressCredentialsProviderTest { val testClient = TestS3Client(expectedCredentials) - DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes).use { provider -> - val attributes = ExecutionContext.build { - this.attributes[S3Attributes.ExpressClient] = testClient - this.attributes[S3Attributes.Bucket] = "bucket" - } + val provider = DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes) - val calls = (1..5).map { - async { provider.resolve(attributes) } - } - calls.awaitAll() + val attributes = ExecutionContext.build { + this.attributes[S3Attributes.ExpressClient] = testClient + this.attributes[S3Attributes.Bucket] = "bucket" } + val calls = (1..5).map { + async { provider.resolve(attributes) } + } + calls.awaitAll() + + // allow the async refresh to initiate before closing the provider + runBlocking { delay(50.milliseconds) } // close the provider, make sure all async refreshes are complete... - runBlocking { delay(10.milliseconds) } + provider.close() + runBlocking { delay(50.milliseconds) } + assertEquals(1, testClient.numCreateSession) } @@ -169,19 +179,23 @@ class DefaultS3ExpressCredentialsProviderTest { // but there should be no crash val testClient = TestS3Client(expectedCredentials, throwExceptionOnBucketNamed = "ExceptionBucket", baseCredentials = Credentials("1", "1", "1")) - DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes).use { provider -> - val attributes = ExecutionContext.build { - this.attributes[S3Attributes.ExpressClient] = testClient - this.attributes[S3Attributes.Bucket] = "ExceptionBucket" - } - provider.resolve(attributes) - - attributes[S3Attributes.Bucket] = "SuccessfulBucket" - provider.resolve(attributes) + val provider = DefaultS3ExpressCredentialsProvider(timeSource, clock, cache, refreshBuffer = 1.minutes) + val attributes = ExecutionContext.build { + this.attributes[S3Attributes.ExpressClient] = testClient + this.attributes[S3Attributes.Bucket] = "ExceptionBucket" } + provider.resolve(attributes) + + attributes[S3Attributes.Bucket] = "SuccessfulBucket" + provider.resolve(attributes) + + // allow the async refresh to initiate before closing the provider + runBlocking { delay(50.milliseconds) } // close the provider, make sure all async refreshes are complete... - runBlocking { delay(10.milliseconds) } + provider.close() + runBlocking { delay(50.milliseconds) } + assertEquals(2, testClient.numCreateSession) } @@ -254,7 +268,6 @@ class DefaultS3ExpressCredentialsProviderTest { override suspend fun createSession(input: CreateSessionRequest): CreateSessionResponse { numCreateSession += 1 - println("i numCreateSession $numCreateSession") throwExceptionOnBucketNamed?.let { if (input.bucket == it) { throw Exception("Failed to create session credentials for bucket: $throwExceptionOnBucketNamed") From 52577d4acc8ee827da264d15703086d69bd0f99a Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 28 Feb 2024 10:48:08 -0500 Subject: [PATCH 136/145] Make interceptors `internal` --- .../services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt | 4 ++-- .../s3/express/S3ExpressDisableChecksumInterceptor.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt index 6172bb7348d..88dd21618c3 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt @@ -17,8 +17,8 @@ internal const val S3_EXPRESS_ENDPOINT_PROPERTY_KEY = "backend" internal const val S3_EXPRESS_ENDPOINT_PROPERTY_VALUE = "S3Express" private const val CRC32_ALGORITHM_NAME = "CRC32" -public class S3ExpressCrc32ChecksumInterceptor( - public val checksumAlgorithmHeaderName: String? = null, +internal class S3ExpressCrc32ChecksumInterceptor( + val checksumAlgorithmHeaderName: String? = null, ) : HttpInterceptor { override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { if (context.executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE) { diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt index 60c3318757b..3b10ad3fa69 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressDisableChecksumInterceptor.kt @@ -15,7 +15,7 @@ import kotlin.coroutines.coroutineContext /** * Disable checksums entirely for s3:UploadPart requests. */ -public class S3ExpressDisableChecksumInterceptor : HttpInterceptor { +internal class S3ExpressDisableChecksumInterceptor : HttpInterceptor { override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext): HttpRequest { if (context.executionContext.getOrNull(AttributeKey(S3_EXPRESS_ENDPOINT_PROPERTY_KEY)) != S3_EXPRESS_ENDPOINT_PROPERTY_VALUE) { return context.protocolRequest From 83a3942ad46a79a7aaa3b2aefc802684f7fda453 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 28 Feb 2024 10:48:14 -0500 Subject: [PATCH 137/145] Fix KDocs --- .../codegen/customization/s3/express/S3ExpressIntegration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt index 586b17bf28f..ce5ded2221a 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/customization/s3/express/S3ExpressIntegration.kt @@ -133,7 +133,7 @@ class S3ExpressIntegration : KotlinIntegration { } /** - * For any operations that may send a checksum, override a user-configured checksum to set CRC32. + * For any operations that require a checksum, set CRC32 if the user has not already configured a checksum. */ private val UseCrc32Checksum = object : ProtocolMiddleware { override val name: String = "UseCrc32Checksum" From 6be45a71dd7ed5dbe883e054466f59f5a77c2070 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 28 Feb 2024 10:48:41 -0500 Subject: [PATCH 138/145] remove unnecessary `client` class variable --- .../express/DefaultS3ExpressCredentialsProvider.kt | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt index bdfd2299adf..d8936f28a3b 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/DefaultS3ExpressCredentialsProvider.kt @@ -41,13 +41,10 @@ internal class DefaultS3ExpressCredentialsProvider( private val credentialsCache: S3ExpressCredentialsCache = S3ExpressCredentialsCache(), private val refreshBuffer: Duration = 1.minutes, ) : CloseableCredentialsProvider, SdkManagedBase(), CoroutineScope { - public constructor() : this(TimeSource.Monotonic) - - private lateinit var client: S3Client override val coroutineContext: CoroutineContext = Job() + CoroutineName("DefaultS3ExpressCredentialsProvider") override suspend fun resolve(attributes: Attributes): Credentials { - client = attributes[S3Attributes.ExpressClient] as S3Client + val client = attributes[S3Attributes.ExpressClient] as S3Client val key = S3ExpressCredentialsCacheKey(attributes[S3Attributes.Bucket], client.config.credentialsProvider.resolve(attributes)) @@ -55,12 +52,12 @@ internal class DefaultS3ExpressCredentialsProvider( ?.takeIf { !it.expiringCredentials.isExpired } ?.also { if (it.expiringCredentials.isExpiringWithin(refreshBuffer)) { - logger.trace { "Credentials for ${key.bucket} are expiring in ${it.expiringCredentials.expiresAt} and are within their refresh window, performing asynchronous refresh..." } + client.logger.trace { "Credentials for ${key.bucket} are expiring in ${it.expiringCredentials.expiresAt} and are within their refresh window, performing asynchronous refresh..." } launch(coroutineContext) { try { it.sfg.singleFlight { createSessionCredentials(key, client) } } catch (e: Exception) { - logger.warn(e) { "Asynchronous refresh for ${key.bucket} failed." } + client.logger.warn(e) { "Asynchronous refresh for ${key.bucket} failed." } } } } @@ -88,6 +85,5 @@ internal class DefaultS3ExpressCredentialsProvider( } @OptIn(ExperimentalApi::class) - internal val logger get() = - client.config.telemetryProvider.loggerProvider.getLogger() + internal val S3Client.logger get() = config.telemetryProvider.loggerProvider.getLogger() } From f6c4fb2b5908ef89dcef340021ac790f9e5d5a1c Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 28 Feb 2024 11:10:59 -0500 Subject: [PATCH 139/145] Validate S3 Express was used for PutObject and add presigner test --- services/s3/e2eTest/src/S3ExpressTest.kt | 61 +++++++++++++++++++----- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/services/s3/e2eTest/src/S3ExpressTest.kt b/services/s3/e2eTest/src/S3ExpressTest.kt index 56905c09af3..b819d3072c6 100644 --- a/services/s3/e2eTest/src/S3ExpressTest.kt +++ b/services/s3/e2eTest/src/S3ExpressTest.kt @@ -5,7 +5,9 @@ package aws.sdk.kotlin.e2etest import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.express.S3_EXPRESS_SESSION_TOKEN_HEADER import aws.sdk.kotlin.services.s3.model.* +import aws.sdk.kotlin.services.s3.presigners.presignPutObject import aws.sdk.kotlin.services.s3.putObject import aws.sdk.kotlin.services.s3.withConfig import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext @@ -19,6 +21,7 @@ import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.TestInstance import kotlin.test.* +import kotlin.time.Duration.Companion.minutes /** * Tests for S3 Express operations @@ -55,22 +58,48 @@ class S3ExpressTest { val keyName = "express.txt" testBuckets.forEach { bucketName -> - client.putObject { - bucket = bucketName - key = keyName - body = ByteStream.fromString(content) + val trackingInterceptor = S3ExpressInvocationTrackingInterceptor() + client.withConfig { + interceptors += trackingInterceptor + }.use { trackingClient -> + trackingClient.putObject { + bucket = bucketName + key = keyName + body = ByteStream.fromString(content) + } + + val req = GetObjectRequest { + bucket = bucketName + key = keyName + } + + val respContent = client.getObject(req) { + it.body?.decodeToString() + } + + assertEquals(content, respContent) + assertEquals(1, trackingInterceptor.s3ExpressInvocations) } + } + } - val req = GetObjectRequest { + @Test + fun testPresignedPutObject() = runTest { + val content = "Presign this!" + val keyName = "express-presigned.txt" + + testBuckets.forEach { bucketName -> + val presigned = client.presignPutObject(PutObjectRequest { bucket = bucketName key = keyName - } + body = ByteStream.fromString(content) + }, 5.minutes) - val respContent = client.getObject(req) { - it.body?.decodeToString() - } + assertTrue(presigned.url.parameters.decodedParameters.contains("X-Amz-Security-Token")) - assertEquals(content, respContent) + // FIXME Presigned requests should use S3 Express Auth Scheme resulting in `X-Amz-S3session-Token` + // https://github.com/awslabs/aws-sdk-kotlin/issues/1236 + assertFalse(presigned.url.parameters.decodedParameters.contains(S3_EXPRESS_SESSION_TOKEN_HEADER)) } } @@ -105,10 +134,20 @@ class S3ExpressTest { } } + private class S3ExpressInvocationTrackingInterceptor : HttpInterceptor { + var s3ExpressInvocations = 0 + + override fun readAfterSigning(context: ProtocolRequestInterceptorContext) { + if (context.protocolRequest.headers.contains(S3_EXPRESS_SESSION_TOKEN_HEADER)) { + s3ExpressInvocations += 1 + } + } + } + private class CRC32ChecksumValidatingInterceptor : HttpInterceptor { override fun readAfterSigning(context: ProtocolRequestInterceptorContext) { val headers = context.protocolRequest.headers - if (headers.contains("X-Amz-S3session-Token")) { + if (headers.contains(S3_EXPRESS_SESSION_TOKEN_HEADER)) { assertTrue(headers.contains("x-amz-checksum-crc32"), "Failed to find x-amz-checksum-crc32 header") assertFalse(headers.contains("Content-MD5"), "Unexpectedly found Content-MD5 header") } From e07509d50563be25d2f859e8f0aade47dc125e0c Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 28 Feb 2024 11:11:36 -0500 Subject: [PATCH 140/145] Downgrade log level --- .../services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt index 88dd21618c3..bad493a2fbd 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressCrc32ChecksumInterceptor.kt @@ -29,7 +29,7 @@ internal class S3ExpressCrc32ChecksumInterceptor( val req = context.protocolRequest.toBuilder() if (!context.executionContext.contains(HttpOperationContext.ChecksumAlgorithm)) { - logger.info { "Checksum is required and not already configured, enabling CRC32 for S3 Express" } + logger.debug { "Checksum is required and not already configured, enabling CRC32 for S3 Express" } // Update the execution context so flexible checksums uses CRC32 context.executionContext[HttpOperationContext.ChecksumAlgorithm] = CRC32_ALGORITHM_NAME From 723781317a451cccc42b48ec93d93a5ccb9371b9 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 28 Feb 2024 13:52:21 -0500 Subject: [PATCH 141/145] ktlint --- services/s3/e2eTest/src/S3ExpressTest.kt | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/services/s3/e2eTest/src/S3ExpressTest.kt b/services/s3/e2eTest/src/S3ExpressTest.kt index b819d3072c6..84538539521 100644 --- a/services/s3/e2eTest/src/S3ExpressTest.kt +++ b/services/s3/e2eTest/src/S3ExpressTest.kt @@ -89,11 +89,14 @@ class S3ExpressTest { val keyName = "express-presigned.txt" testBuckets.forEach { bucketName -> - val presigned = client.presignPutObject(PutObjectRequest { - bucket = bucketName - key = keyName - body = ByteStream.fromString(content) - }, 5.minutes) + val presigned = client.presignPutObject( + PutObjectRequest { + bucket = bucketName + key = keyName + body = ByteStream.fromString(content) + }, + 5.minutes, + ) assertTrue(presigned.url.parameters.decodedParameters.contains("X-Amz-Security-Token")) From 624ac0ef969988aa06aa51afa541da4e66c25700 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 28 Feb 2024 13:52:52 -0500 Subject: [PATCH 142/145] make const header val `internal` --- .../aws/sdk/kotlin/services/s3/express/S3ExpressHttpSigner.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressHttpSigner.kt b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressHttpSigner.kt index 29224098c4b..288ed5e6f69 100644 --- a/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressHttpSigner.kt +++ b/services/s3/common/src/aws/sdk/kotlin/services/s3/express/S3ExpressHttpSigner.kt @@ -11,7 +11,7 @@ import aws.smithy.kotlin.runtime.http.auth.HttpSigner import aws.smithy.kotlin.runtime.http.auth.SignHttpRequest import aws.smithy.kotlin.runtime.http.request.header -private const val S3_EXPRESS_SESSION_TOKEN_HEADER = "X-Amz-S3session-Token" +internal const val S3_EXPRESS_SESSION_TOKEN_HEADER = "X-Amz-S3session-Token" private const val SESSION_TOKEN_HEADER = "X-Amz-Security-Token" /** From 5c2dfe49bd0e5e30741e5bc31b27b15e6f3f326e Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 28 Feb 2024 14:34:49 -0500 Subject: [PATCH 143/145] Add S3 Express benchmarks --- tests/benchmarks/service-benchmarks/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/benchmarks/service-benchmarks/README.md b/tests/benchmarks/service-benchmarks/README.md index 5d13740fd5e..1168540cf80 100644 --- a/tests/benchmarks/service-benchmarks/README.md +++ b/tests/benchmarks/service-benchmarks/README.md @@ -51,6 +51,19 @@ The following benchmark run serves as a baseline for future runs: | —GetEndpoint | | 555 | 0.220 | 0.401 | 0.406 | 0.452 | 0.506 | 6.606 | | —PutEvents | | 415 | 0.242 | 0.400 | 0.420 | 0.466 | 0.619 | 2.762 | +### S3 Express +S3 Express benchmarks were ran separately. + +| Hardware type | Operating system | SDK version | +|----------------|------------------|-----------------| +| EC2 m5.4xlarge | Amazon Linux 2023 | 1.0.66 | + +| | Overhead (ms) | n | min | avg | med | p90 | p99 | max | +| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| **S3Express** | | | | | | | | | +| —PutObject | | 2095 | 0.677 | 0.744 | 0.725 | 0.762 | 0.842 | 7.532 | +| —GetObject | | 3466 | 0.281 | 0.321 | 0.302 | 0.344 | 0.373 | 8.932 | + ## Methodology This section describes how the benchmarks actually work at a high level: From fca9d7ad7d7149f5381c6a25ab1f1ad10d541ee2 Mon Sep 17 00:00:00 2001 From: Matas Lauzadis Date: Wed, 28 Feb 2024 14:42:59 -0500 Subject: [PATCH 144/145] Use `smithy.client.call.attempt_duration` metric --- tests/benchmarks/service-benchmarks/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/benchmarks/service-benchmarks/README.md b/tests/benchmarks/service-benchmarks/README.md index 1168540cf80..fd34d54ed41 100644 --- a/tests/benchmarks/service-benchmarks/README.md +++ b/tests/benchmarks/service-benchmarks/README.md @@ -58,11 +58,11 @@ S3 Express benchmarks were ran separately. |----------------|------------------|-----------------| | EC2 m5.4xlarge | Amazon Linux 2023 | 1.0.66 | -| | Overhead (ms) | n | min | avg | med | p90 | p99 | max | -| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| **S3Express** | | | | | | | | | -| —PutObject | | 2095 | 0.677 | 0.744 | 0.725 | 0.762 | 0.842 | 7.532 | -| —GetObject | | 3466 | 0.281 | 0.321 | 0.302 | 0.344 | 0.373 | 8.932 | +| | E2E Duration (ms) | n | min | avg | med | p90 | p99 | max | +| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| **S3Express** | | | | | | | | | +| —PutObject | | 1950 | 7.240 | 7.487 | 7.455 | 7.617 | 7.886 | 21.096 | +| —GetObject | | 3402 | 4.049 | 4.188 | 4.141 | 4.243 | 4.470 | 20.537 | ## Methodology From 84e166504654237c537c5d94c0ac9df4cba9f62a Mon Sep 17 00:00:00 2001 From: Matas Date: Wed, 28 Feb 2024 13:36:27 -0800 Subject: [PATCH 145/145] Bump to latest version of smithy-kotlin --- gradle/libs.versions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 88a30d55c61..f19bf376040 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -9,8 +9,8 @@ coroutines-version = "1.7.3" atomicfu-version = "0.23.1" # smithy-kotlin codegen and runtime are versioned separately -smithy-kotlin-runtime-version = "1.0.16-SNAPSHOT" -smithy-kotlin-codegen-version = "0.30.17-SNAPSHOT" +smithy-kotlin-runtime-version = "1.0.16" +smithy-kotlin-codegen-version = "0.30.17" # codegen smithy-version = "1.42.0"