From ec0850b19c0b2772b6b8f6d7bcdcab0fd9aa711b Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Mon, 9 Jun 2025 11:52:55 -0400 Subject: [PATCH 1/4] track and return content bytes transferred --- .../api/aws-signing-common.api | 2 ++ .../auth/awssigning/AwsChunkedSource.kt | 9 ++++++- .../awssigning/internal/AwsChunkedReader.kt | 26 ++++++++++++++----- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/runtime/auth/aws-signing-common/api/aws-signing-common.api b/runtime/auth/aws-signing-common/api/aws-signing-common.api index c9353509be..1096f087ce 100644 --- a/runtime/auth/aws-signing-common/api/aws-signing-common.api +++ b/runtime/auth/aws-signing-common/api/aws-signing-common.api @@ -23,7 +23,9 @@ public final class aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource : public fun (Laws/smithy/kotlin/runtime/io/SdkSource;Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner;Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigningConfig;[BLaws/smithy/kotlin/runtime/http/DeferredHeaders;)V public synthetic fun (Laws/smithy/kotlin/runtime/io/SdkSource;Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner;Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigningConfig;[BLaws/smithy/kotlin/runtime/http/DeferredHeaders;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public fun close ()V + public final fun getContentBytesTransferred ()J public fun read (Laws/smithy/kotlin/runtime/io/SdkBuffer;J)J + public final fun setContentBytesTransferred (J)V } public final class aws/smithy/kotlin/runtime/auth/awssigning/AwsSignatureType : java/lang/Enum { diff --git a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt index edfe598404..3520e92069 100644 --- a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt +++ b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt @@ -40,6 +40,9 @@ public class AwsChunkedSource( trailingHeaders, ) + // The number of bytes transferred without chunk metadata + public var contentBytesTransferred: Long = 0L + override fun read(sink: SdkBuffer, limit: Long): Long { require(limit >= 0L) { "Invalid limit ($limit) must be >= 0L" } // COROUTINE SAFETY: runBlocking is allowed here because SdkSource is a synchronous blocking interface @@ -47,7 +50,11 @@ public class AwsChunkedSource( chunkReader.ensureValidChunk() } if (!isChunkValid) return -1L - return chunkReader.chunk.read(sink, limit) + + val actualBytesTransferred = chunkReader.chunk.read(sink, limit) + contentBytesTransferred = actualBytesTransferred - chunkReader.chunkMetadataBytes + + return actualBytesTransferred } override fun close() { diff --git a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt index 05061cfd98..94d34fdbfa 100644 --- a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt +++ b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt @@ -57,6 +57,13 @@ internal class AwsChunkedReader( */ internal var hasLastChunkBeenSent: Boolean = false + /** + * Tracks the number of bytes used for chunk metadata. + * This includes chunk headers, terminators (CRLF), and trailers. + * Used to calculate the actual payload bytes transferred by subtracting metadata bytes from the total bytes read. + */ + internal var chunkMetadataBytes: Long = 0L + /** * Ensures that the internal [chunk] is valid for reading. If it's not valid, try to load the next chunk. Note that * this function will suspend until the whole chunk has been loaded. @@ -65,7 +72,11 @@ internal class AwsChunkedReader( */ internal suspend fun ensureValidChunk(): Boolean { // check if the current chunk is still valid - if (chunk.size > 0L) return true + if (chunk.size > 0L) { + // // Reset metadata bytes counter as only first read of a chunk contains metadata + chunkMetadataBytes = 0L + return true + } // if not, try to fetch a new chunk val nextChunk = when { @@ -80,9 +91,10 @@ internal class AwsChunkedReader( next } } - + val preTerminatorChunkSize = nextChunk?.size ?: 0L nextChunk?.writeUtf8("\r\n") // terminating CRLF to signal end of chunk - + val chunkSizeWithTerminator = nextChunk?.size ?: 0L + chunkMetadataBytes += chunkSizeWithTerminator - preTerminatorChunkSize // transfer all segments to the working chunk nextChunk?.let { chunk.writeAll(it) } @@ -96,12 +108,14 @@ internal class AwsChunkedReader( private suspend fun getFinalChunk(): SdkBuffer { // empty chunk val lastChunk = checkNotNull(if (signingConfig.isUnsigned) getUnsignedChunk(SdkBuffer()) else getSignedChunk(SdkBuffer())) - + val preTrailerChunkSize = lastChunk.size // + any trailers if (!trailingHeaders.isEmpty()) { val trailingHeaderChunk = getTrailingHeadersChunk(trailingHeaders.toHeaders()) lastChunk.writeAll(trailingHeaderChunk) } + val trailersSize = lastChunk.size - preTrailerChunkSize + chunkMetadataBytes += trailersSize return lastChunk } @@ -155,7 +169,7 @@ internal class AwsChunkedReader( write(chunkSignature) writeUtf8("\r\n") } - + chunkMetadataBytes += signedChunk.size // append the body signedChunk.write(chunkBody) @@ -183,7 +197,7 @@ internal class AwsChunkedReader( writeUtf8("\r\n") writeAll(bodyBuffer) // append the body } - + chunkMetadataBytes += unsignedChunk.size - bodyBuffer.size return unsignedChunk } From 2f844cb622a5592136403c38bbf1fe198bc4ca06 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Mon, 9 Jun 2025 12:05:43 -0400 Subject: [PATCH 2/4] style --- .../kotlin/runtime/auth/awssigning/AwsChunkedSource.kt | 5 ++++- .../runtime/auth/awssigning/internal/AwsChunkedReader.kt | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt index 3520e92069..13dff20f3c 100644 --- a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt +++ b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt @@ -40,7 +40,10 @@ public class AwsChunkedSource( trailingHeaders, ) - // The number of bytes transferred without chunk metadata + /** + * Tracks the content bytes transferred, excluding chunk metadata. + * This public property can be accessed to monitor file transfer progress. + */ public var contentBytesTransferred: Long = 0L override fun read(sink: SdkBuffer, limit: Long): Long { diff --git a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt index 94d34fdbfa..81c3a92666 100644 --- a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt +++ b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt @@ -73,7 +73,7 @@ internal class AwsChunkedReader( internal suspend fun ensureValidChunk(): Boolean { // check if the current chunk is still valid if (chunk.size > 0L) { - // // Reset metadata bytes counter as only first read of a chunk contains metadata + // reset metadata bytes counter as only first read of a chunk contains metadata chunkMetadataBytes = 0L return true } From 2c1dbd2e45817f7bfb397bc9633bccb2099c4e92 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Mon, 9 Jun 2025 13:44:07 -0400 Subject: [PATCH 3/4] wording --- .../kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt index 81c3a92666..b34e48a062 100644 --- a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt +++ b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt @@ -60,7 +60,7 @@ internal class AwsChunkedReader( /** * Tracks the number of bytes used for chunk metadata. * This includes chunk headers, terminators (CRLF), and trailers. - * Used to calculate the actual payload bytes transferred by subtracting metadata bytes from the total bytes read. + * Used to calculate the content bytes transferred by subtracting metadata bytes from the total bytes read. */ internal var chunkMetadataBytes: Long = 0L From b04cf1b8b32970a64981aa68346ba99359c0b305 Mon Sep 17 00:00:00 2001 From: Xinsong Cui Date: Mon, 9 Jun 2025 13:45:08 -0400 Subject: [PATCH 4/4] wording --- .../kotlin/runtime/auth/awssigning/AwsChunkedSource.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt index 13dff20f3c..ebe6162a2e 100644 --- a/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt +++ b/runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt @@ -54,10 +54,10 @@ public class AwsChunkedSource( } if (!isChunkValid) return -1L - val actualBytesTransferred = chunkReader.chunk.read(sink, limit) - contentBytesTransferred = actualBytesTransferred - chunkReader.chunkMetadataBytes + val totalBytesTransferred = chunkReader.chunk.read(sink, limit) + contentBytesTransferred = totalBytesTransferred - chunkReader.chunkMetadataBytes - return actualBytesTransferred + return totalBytesTransferred } override fun close() {