toTempFile(InputStream inputStream, HttpHeaders httpHe
wrappedStream.transferTo(os);
ChecksumAlgorithm algorithmFromSdk = checksumAlgorithmFromSdk(httpHeaders);
if (algorithmFromSdk != null
- && wrappedStream instanceof AwsChunkedInputStream awsInputStream) {
+ && wrappedStream instanceof AbstractAwsInputStream awsInputStream) {
return Pair.of(tempFile, awsInputStream.getChecksum());
}
return Pair.of(tempFile, null);
@@ -292,16 +294,23 @@ public void verifyChecksum(Path path, String checksum, ChecksumAlgorithm checksu
InputStream wrapStream(InputStream dataStream, HttpHeaders headers) {
var lengthHeader = headers.getFirst(X_AMZ_DECODED_CONTENT_LENGTH);
- var trailHeader = headers.getOrEmpty(X_AMZ_TRAILER);
- var hasChecksum = trailHeader.stream().anyMatch(h -> h.contains(X_AMZ_CHECKSUM));
var length = lengthHeader == null ? -1 : Long.parseLong(lengthHeader);
- if (isChunkedAndV4Signed(headers) || isChunked(headers)) {
- return new AwsChunkedInputStream(dataStream, length, hasChecksum);
+ boolean hasChecksum = hasChecksum(headers);
+ if (isChunkedAndV4Signed(headers)) {
+ return new AwsChunkedDecodingChecksumInputStream(dataStream, length);
+ } else if (isChunked(headers)) {
+ return new AwsUnsignedChunkedDecodingChecksumInputStream(dataStream, length);
} else {
return dataStream;
}
}
+ private boolean hasChecksum(HttpHeaders headers) {
+ var trailHeader = headers.getOrEmpty(X_AMZ_TRAILER);
+ return isChunkedAndV4Signed(headers)
+ || trailHeader.stream().anyMatch(h -> h.contains(X_AMZ_CHECKSUM));
+ }
+
public void verifyMd5(Path input, String contentMd5) {
try {
try (var stream = Files.newInputStream(input)) {
diff --git a/server/src/main/java/com/adobe/testing/s3mock/util/AbstractAwsInputStream.java b/server/src/main/java/com/adobe/testing/s3mock/util/AbstractAwsInputStream.java
new file mode 100644
index 000000000..3dace9e39
--- /dev/null
+++ b/server/src/main/java/com/adobe/testing/s3mock/util/AbstractAwsInputStream.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2017-2024 Adobe.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.adobe.testing.s3mock.util;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.adobe.testing.s3mock.dto.ChecksumAlgorithm;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+public abstract class AbstractAwsInputStream extends InputStream {
+ protected static final byte[] CRLF = "\r\n".getBytes(UTF_8);
+ protected static final byte[] DELIMITER = ";".getBytes(UTF_8);
+ protected static final byte[] CHECKSUM_HEADER = "x-amz-checksum-".getBytes(UTF_8);
+ protected long readDecodedLength = 0L;
+ protected final InputStream source;
+ protected long chunkLength = 0L;
+ protected String checksum;
+ protected ChecksumAlgorithm algorithm;
+ protected int chunks = 0;
+ /**
+ * That's the max chunk buffer size used in the AWS implementation.
+ */
+ private static final int MAX_CHUNK_SIZE = 256 * 1024;
+ private final ByteBuffer byteBuffer = ByteBuffer.allocate(MAX_CHUNK_SIZE);
+ protected final long decodedLength;
+
+ protected AbstractAwsInputStream(InputStream source, long decodedLength) {
+ this.source = new BufferedInputStream(source);
+ this.decodedLength = decodedLength;
+ }
+
+ @Override
+ public void close() throws IOException {
+ source.close();
+ }
+
+ /**
+ * Reads this stream until the byte sequence was found.
+ *
+ * @param endSequence The byte sequence to look for in the stream. The source stream is read
+ * until the last bytes read are equal to this sequence.
+ *
+ * @return The bytes read before the end sequence started.
+ */
+ protected byte[] readUntil(final byte[] endSequence) throws IOException {
+ byteBuffer.clear();
+ while (!endsWith(byteBuffer.asReadOnlyBuffer(), endSequence)) {
+ var c = source.read();
+ if (c < 0) {
+ return new byte[0];
+ }
+
+ var unsigned = (byte) (c & 0xFF);
+ byteBuffer.put(unsigned);
+ }
+
+ var result = new byte[byteBuffer.position() - endSequence.length];
+ byteBuffer.rewind();
+ byteBuffer.get(result);
+ return result;
+ }
+
+ protected boolean endsWith(final ByteBuffer buffer, final byte[] endSequence) {
+ var pos = buffer.position();
+ if (pos >= endSequence.length) {
+ for (var i = 0; i < endSequence.length; i++) {
+ if (buffer.get(pos - endSequence.length + i) != endSequence[i]) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ return false;
+ }
+
+ protected void setChunkLength(byte[] hexLengthBytes) {
+ chunkLength = Long.parseLong(new String(hexLengthBytes, UTF_8).trim(), 16);
+ }
+
+ protected void extractAlgorithmAndChecksum() throws IOException {
+ if (algorithm == null && checksum == null) {
+ readUntil(CHECKSUM_HEADER);
+ var typeAndChecksum = readUntil(CRLF);
+ var typeAndChecksumString = new String(typeAndChecksum);
+ if (!typeAndChecksumString.isBlank()) {
+ var split = typeAndChecksumString.split(":");
+ var type = split[0];
+ algorithm = ChecksumAlgorithm.fromString(type);
+ checksum = split[1];
+ }
+ }
+ }
+
+ public String getChecksum() {
+ return checksum;
+ }
+
+ public ChecksumAlgorithm getAlgorithm() {
+ return algorithm;
+ }
+}
diff --git a/server/src/main/java/com/adobe/testing/s3mock/util/AwsChunkedDecodingChecksumInputStream.java b/server/src/main/java/com/adobe/testing/s3mock/util/AwsChunkedDecodingChecksumInputStream.java
new file mode 100644
index 000000000..7b55b9145
--- /dev/null
+++ b/server/src/main/java/com/adobe/testing/s3mock/util/AwsChunkedDecodingChecksumInputStream.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2017-2024 Adobe.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.adobe.testing.s3mock.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Merges chunks from AWS chunked AwsChunkedEncodingInputStream, skipping V4 style signing metadata.
+ * The checksum is optionally included in the stream as part of the "trail headers"
+ * after the last chunk.
+ *
+ * The original stream looks like this:
+ *
+ *
+ * 24;chunk-signature=312a41de690364ad6d17629d1e026c448e78abd328f1602276fdd2c3f928d100
+ * ## sample test file ##
+ *
+ * demo=content
+ * 0;chunk-signature=4d2b448448f29b473beb81340f5a3d6c9468e4fbb9ac761cfab63846919011fb
+ * x-amz-checksum-sha256:1VcEifAruhjVvjzul4sC0B1EmlUdzqvsp6BP0KSVdTE=
+ *
+ *
+ * The format of each chunk of data is:
+ *
+ *
+ * [hex-encoded-number-of-bytes-in-chunk];chunk-signature=[sha256-signature][EOL]
+ * [payload-bytes-of-this-chunk][EOL]
+ *
+ *
+ * The format of the full payload is:
+ *
+ *
+ * [hex-encoded-number-of-bytes-in-chunk];chunk-signature=[sha256-signature][EOL]
+ * [payload-bytes-of-this-chunk][EOL]
+ * 0;chunk-signature=[sha256-signature][EOL]
+ * x-amz-checksum-[checksum-algorithm]:[checksum][EOL]
+ * [other trail headers]
+ *
+ *
+ * @see
+ *
+ * AwsChunkedEncodingInputStream
+ */
+public class AwsChunkedDecodingChecksumInputStream extends AbstractAwsInputStream {
+
+ public AwsChunkedDecodingChecksumInputStream(InputStream source, long decodedLength) {
+ super(source, decodedLength);
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (chunkLength == 0L) {
+ //try to read chunk length
+ var hexLengthBytes = readHexLength();
+ if (hexLengthBytes.length == 0) {
+ return -1;
+ }
+
+ setChunkLength(hexLengthBytes);
+
+ if (chunkLength == 0L) {
+ //chunk length found, but was "0". Try and find the checksum.
+ extractAlgorithmAndChecksum();
+ return -1;
+ }
+
+ chunks++;
+ readUntil(CRLF);
+ }
+
+ readDecodedLength++;
+ chunkLength--;
+
+ return source.read();
+ }
+
+ private byte[] readHexLength() throws IOException {
+ return readUntil(DELIMITER);
+ }
+}
diff --git a/server/src/main/java/com/adobe/testing/s3mock/util/AwsChunkedInputStream.java b/server/src/main/java/com/adobe/testing/s3mock/util/AwsChunkedInputStream.java
deleted file mode 100644
index df52323b2..000000000
--- a/server/src/main/java/com/adobe/testing/s3mock/util/AwsChunkedInputStream.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Copyright 2017-2024 Adobe.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.adobe.testing.s3mock.util;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.adobe.testing.s3mock.dto.ChecksumAlgorithm;
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * This handles both signed and unsigned chunked encodings.
- *
- * SIGNED:
- * Merges chunks from AWS chunked AwsChunkedEncodingInputStream, skipping V4 style signing metadata.
- * The checksum is optionally included in the stream as part of the "trail headers"
- * after the last chunk.
- *
- * The original stream looks like this:
- *
- *
- * 24;chunk-signature=312a41de690364ad6d17629d1e026c448e78abd328f1602276fdd2c3f928d100
- * ## sample test file ##
- *
- * demo=content
- * 0;chunk-signature=4d2b448448f29b473beb81340f5a3d6c9468e4fbb9ac761cfab63846919011fb
- * x-amz-checksum-sha256:1VcEifAruhjVvjzul4sC0B1EmlUdzqvsp6BP0KSVdTE=
- *
- *
- * The format of each chunk of data is:
- *
- *
- * [hex-encoded-number-of-bytes-in-chunk];chunk-signature=[sha256-signature][EOL]
- * [payload-bytes-of-this-chunk][EOL]
- *
- *
- * The format of the full payload is:
- *
- *
- * [hex-encoded-number-of-bytes-in-chunk];chunk-signature=[sha256-signature][EOL]
- * [payload-bytes-of-this-chunk][EOL]
- * 0;chunk-signature=[sha256-signature][EOL]
- * x-amz-checksum-[checksum-algorithm]:[checksum][EOL]
- * [other trail headers]
- *
- *
- * UNSIGNED:
- * Merges chunks from AWS chunked AwsUnsignedChunkedEncodingInputStream.
- * The checksum is optionally included in the stream as part of the "trail headers"
- * after the last chunk.
- *
- * The original stream looks like this:
- *
- *
- * 24
- * ## sample test file ##
- *
- * demo=content
- * 0
- * x-amz-checksum-sha256:1VcEifAruhjVvjzul4sC0B1EmlUdzqvsp6BP0KSVdTE=
- *
- *
- * The format of each chunk of data is:
- *
- *
- * [hex-encoded-number-of-bytes-in-chunk][EOL]
- * [payload-bytes-of-this-chunk][EOL]
- *
- *
- * The format of the full payload is:
- *
- *
- * [hex-encoded-number-of-bytes-in-chunk][EOL]
- * [payload-bytes-of-this-chunk][EOL]
- * 0[EOL]
- * x-amz-checksum-[checksum-algoritm]:[checksum][EOL]
- * [other trail headers]
- *
- *
- * @see
- *
- * AwsUnsignedChunkedEncodingInputStream
- *
- * @see
- *
- * AwsChunkedEncodingInputStream
- */
-public class AwsChunkedInputStream extends InputStream {
- protected long readDecodedLength = 0L;
- protected BufferedInputStream source;
- protected BufferedReader reader;
- protected long chunkLength = 0L;
- protected int chunks = 0;
- protected String checksum;
- protected ChecksumAlgorithm algorithm;
- protected final boolean hasChecksum;
- protected final long decodedLength;
-
- public AwsChunkedInputStream(InputStream source, long decodedLength, boolean hasChecksum) {
- this.reader = new BufferedReader(new InputStreamReader(source, UTF_8));
- this.decodedLength = decodedLength;
- this.hasChecksum = hasChecksum;
- }
-
- @Override
- public int read() throws IOException {
- if (chunkLength == -1L) {
- //stream end was marked after last chunk was read.
- // See below.
- // Always return -1.
- return -1;
- }
- if (chunkLength == 0L) {
- //try to read chunk length
- var hexLength = readHexLength();
- if (hexLength == null) {
- return -1;
- }
- setChunkLength(hexLength);
-
- if (chunkLength == 0L) {
- //chunk length found, but was "0".
- // This marks the end of the payload and the beginning of the trail headers.
- // Extract checksum, if available.
- extractAlgorithmAndChecksum();
- // make sure that no more bytes are read from the stream, see first line.
- chunkLength = -1L;
- return -1;
- }
- chunks++;
- }
-
- readDecodedLength++;
- chunkLength--;
-
- return reader.read();
- }
-
- @Override
- public void close() throws IOException {
- reader.close();
- }
-
- public String getChecksum() {
- return checksum;
- }
-
- public ChecksumAlgorithm getAlgorithm() {
- return algorithm;
- }
-
- private void setChunkLength(String hexLength) {
- chunkLength = Long.parseLong(hexLength.trim(), 16);
- }
-
- private void extractAlgorithmAndChecksum() throws IOException {
- if (hasChecksum) {
- var typeAndChecksumString = reader.readLine();
- if (typeAndChecksumString != null && !typeAndChecksumString.isBlank()) {
- var split = typeAndChecksumString.split(":");
- var type = split[0];
- algorithm = ChecksumAlgorithm.fromHeader(type);
- checksum = split[1];
- }
- }
- }
-
- private String readHexLength() throws IOException {
- var hexLength = reader.readLine();
- if (hexLength != null && hexLength.isBlank()) {
- //skip empty line if present
- hexLength = reader.readLine();
- }
- if (hexLength != null && !hexLength.isBlank()) {
- //remove chunk signature, if present
- hexLength = hexLength.split(";")[0];
- }
- return hexLength;
- }
-}
diff --git a/server/src/main/java/com/adobe/testing/s3mock/util/AwsUnsignedChunkedDecodingChecksumInputStream.java b/server/src/main/java/com/adobe/testing/s3mock/util/AwsUnsignedChunkedDecodingChecksumInputStream.java
new file mode 100644
index 000000000..ba06f280b
--- /dev/null
+++ b/server/src/main/java/com/adobe/testing/s3mock/util/AwsUnsignedChunkedDecodingChecksumInputStream.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2017-2024 Adobe.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.adobe.testing.s3mock.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Merges chunks from AWS chunked AwsUnsignedChunkedEncodingInputStream.
+ * The checksum is optionally included in the stream as part of the "trail headers"
+ * after the last chunk.
+ *
+ * The original stream looks like this:
+ *
+ *
+ * 24
+ * ## sample test file ##
+ *
+ * demo=content
+ * 0
+ * x-amz-checksum-sha256:1VcEifAruhjVvjzul4sC0B1EmlUdzqvsp6BP0KSVdTE=
+ *
+ *
+ * The format of each chunk of data is:
+ *
+ *
+ * [hex-encoded-number-of-bytes-in-chunk][EOL]
+ * [payload-bytes-of-this-chunk][EOL]
+ *
+ *
+ * The format of the full payload is:
+ *
+ *
+ * [hex-encoded-number-of-bytes-in-chunk][EOL]
+ * [payload-bytes-of-this-chunk][EOL]
+ * 0[EOL]
+ * x-amz-checksum-[checksum-algoritm]:[checksum][EOL]
+ * [other trail headers]
+ *
+ *
+ * @see
+ *
+ * AwsUnsignedChunkedEncodingInputStream
+ */
+public class AwsUnsignedChunkedDecodingChecksumInputStream extends AbstractAwsInputStream {
+
+ public AwsUnsignedChunkedDecodingChecksumInputStream(InputStream source, long decodedLength) {
+ super(source, decodedLength);
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (chunkLength == 0L) {
+ //try to read chunk length
+ var hexLengthBytes = readHexlength();
+ if (hexLengthBytes.length == 0) {
+ return -1;
+ }
+
+ setChunkLength(hexLengthBytes);
+
+ if (chunkLength == 0L) {
+ //chunk length found, but was "0". Try and find the checksum.
+ extractAlgorithmAndChecksum();
+ return -1;
+ }
+
+ chunks++;
+ }
+
+ readDecodedLength++;
+ chunkLength--;
+
+ return source.read();
+ }
+
+ private byte[] readHexlength() throws IOException {
+ var hexLengthBytes = readUntil(CRLF);
+ if (hexLengthBytes.length == 0) {
+ hexLengthBytes = readUntil(CRLF);
+ }
+ return hexLengthBytes;
+ }
+}
diff --git a/server/src/test/java/com/adobe/testing/s3mock/util/AwsChunkedDecodingChecksumInputStreamTest.java b/server/src/test/java/com/adobe/testing/s3mock/util/AwsChunkedDecodingChecksumInputStreamTest.java
new file mode 100644
index 000000000..275976912
--- /dev/null
+++ b/server/src/test/java/com/adobe/testing/s3mock/util/AwsChunkedDecodingChecksumInputStreamTest.java
@@ -0,0 +1,172 @@
+/*
+ * Copyright 2017-2024 Adobe.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.adobe.testing.s3mock.util;
+
+import static com.adobe.testing.s3mock.dto.ChecksumAlgorithm.SHA256;
+import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_SHA256;
+import static com.adobe.testing.s3mock.util.DigestUtil.checksumFor;
+import static com.adobe.testing.s3mock.util.TestUtil.getFileFromClasspath;
+import static java.nio.file.Files.newInputStream;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.adobe.testing.s3mock.dto.ChecksumAlgorithm;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.awssdk.auth.signer.internal.chunkedencoding.AwsS3V4ChunkSigner;
+import software.amazon.awssdk.auth.signer.internal.chunkedencoding.AwsSignedChunkedEncodingInputStream;
+import software.amazon.awssdk.core.checksums.Algorithm;
+import software.amazon.awssdk.core.checksums.SdkChecksum;
+import software.amazon.awssdk.core.internal.chunked.AwsChunkedEncodingConfig;
+
+class AwsChunkedDecodingChecksumInputStreamTest {
+
+ @Test
+ void testDecode_checksum(TestInfo testInfo) throws IOException {
+ doTest(testInfo, "sampleFile.txt", X_AMZ_CHECKSUM_SHA256, Algorithm.SHA256,
+ "1VcEifAruhjVvjzul4sC0B1EmlUdzqvsp6BP0KSVdTE=", SHA256, 1);
+ doTest(testInfo, "sampleFile_large.txt", X_AMZ_CHECKSUM_SHA256, Algorithm.SHA256,
+ "Y8S4/uAGut7vjdFZQjLKZ7P28V9EPWb4BIoeniuM0mY=", SHA256, 16);
+ }
+
+ @Test
+ void testDecode_noChecksum(TestInfo testInfo) throws IOException {
+ doTest(testInfo, "sampleFile.txt", 1);
+ doTest(testInfo, "sampleFile_large.txt", 16);
+ }
+
+ void doTest(TestInfo testInfo, String fileName, int chunks) throws IOException {
+ doTest(testInfo, fileName, null, null, null, null, chunks);
+ }
+
+ void doTest(TestInfo testInfo, String fileName, String header, Algorithm algorithm,
+ String checksum, ChecksumAlgorithm checksumAlgorithm, int chunks) throws IOException {
+ File sampleFile = getFileFromClasspath(testInfo, fileName);
+ AwsSignedChunkedEncodingInputStream.Builder builder = AwsSignedChunkedEncodingInputStream
+ .builder()
+ .inputStream(Files.newInputStream(sampleFile.toPath()));
+ if (algorithm != null) {
+ builder.sdkChecksum(SdkChecksum.forAlgorithm(algorithm));
+ }
+ InputStream chunkedEncodingInputStream = builder
+ .checksumHeaderForTrailer(header)
+ //force chunks in the inputstream
+ .awsChunkedEncodingConfig(AwsChunkedEncodingConfig.builder().chunkSize(4000).build())
+ .awsChunkSigner(new AwsS3V4ChunkSigner("signingKey".getBytes(),
+ "dateTime",
+ "keyPath"))
+ .build();
+
+ long decodedLength = sampleFile.length();
+ AwsChunkedDecodingChecksumInputStream iut = new
+ AwsChunkedDecodingChecksumInputStream(chunkedEncodingInputStream, decodedLength);
+ assertThat(iut).hasSameContentAs(Files.newInputStream(sampleFile.toPath()));
+ assertThat(iut.getAlgorithm()).isEqualTo(checksumAlgorithm);
+ assertThat(iut.getChecksum()).isEqualTo(checksum);
+ assertThat(iut.decodedLength).isEqualTo(decodedLength);
+ assertThat(iut.readDecodedLength).isEqualTo(decodedLength);
+ assertThat(iut.chunks).isEqualTo(chunks);
+ }
+
+ @ParameterizedTest
+ @MethodSource("algorithms")
+ void testDecode_signed_checksum(Algorithm algorithm, TestInfo testInfo) throws IOException {
+ ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.fromString(algorithm.toString());
+ String header = HeaderUtil.mapChecksumToHeader(checksumAlgorithm);
+ doTestSigned(getFileFromClasspath(testInfo, "sampleFile.txt"),
+ 1,
+ header,
+ SdkChecksum.forAlgorithm(algorithm),
+ checksumFor(
+ getFileFromClasspath(testInfo, "sampleFile.txt").toPath(), algorithm),
+ checksumAlgorithm
+ );
+ doTestSigned(getFileFromClasspath(testInfo, "sampleFile_large.txt"),
+ 16,
+ header,
+ SdkChecksum.forAlgorithm(algorithm),
+ checksumFor(
+ getFileFromClasspath(testInfo, "sampleFile_large.txt").toPath(), algorithm),
+ checksumAlgorithm
+ );
+ doTestSigned(getFileFromClasspath(testInfo, "test-image-small.png"),
+ 9,
+ header,
+ SdkChecksum.forAlgorithm(algorithm),
+ checksumFor(
+ getFileFromClasspath(testInfo, "test-image-small.png").toPath(), algorithm),
+ checksumAlgorithm
+ );
+ doTestSigned(getFileFromClasspath(testInfo, "test-image.png"),
+ 17,
+ header,
+ SdkChecksum.forAlgorithm(algorithm),
+ checksumFor(
+ getFileFromClasspath(testInfo, "test-image.png").toPath(), algorithm),
+ checksumAlgorithm
+ );
+ }
+
+ @Test
+ void testDecode_signed_noChecksum(TestInfo testInfo) throws IOException {
+ doTestSigned(getFileFromClasspath(testInfo, "sampleFile.txt"), 1);
+ doTestSigned(getFileFromClasspath(testInfo, "sampleFile_large.txt"), 16);
+ doTestSigned(getFileFromClasspath(testInfo, "test-image-small.png"), 9);
+ doTestSigned(getFileFromClasspath(testInfo, "test-image.png"), 17);
+ }
+
+ void doTestSigned(File input, int chunks) throws IOException {
+ doTestSigned(input, chunks, null, null, null, null);
+ }
+
+ void doTestSigned(File input, int chunks, String header, SdkChecksum algorithm,
+ String checksum, ChecksumAlgorithm checksumAlgorithm) throws IOException {
+ InputStream chunkedEncodingInputStream = AwsSignedChunkedEncodingInputStream
+ .builder()
+ .inputStream(newInputStream(input.toPath()))
+ .sdkChecksum(algorithm)
+ .checksumHeaderForTrailer(header)
+ //force chunks in the inputstream
+ .awsChunkedEncodingConfig(AwsChunkedEncodingConfig.builder().chunkSize(4000).build())
+ .awsChunkSigner(new AwsS3V4ChunkSigner("signingKey".getBytes(),
+ "dateTime",
+ "keyPath"))
+ .build();
+
+ long decodedLength = input.length();
+ AwsChunkedDecodingChecksumInputStream iut =
+ new AwsChunkedDecodingChecksumInputStream(chunkedEncodingInputStream, decodedLength);
+
+ assertThat(iut).hasSameContentAs(newInputStream(input.toPath()));
+ assertThat(iut.getAlgorithm()).isEqualTo(checksumAlgorithm);
+ assertThat(iut.getChecksum()).isEqualTo(checksum);
+ assertThat(iut.decodedLength).isEqualTo(decodedLength);
+ assertThat(iut.readDecodedLength).isEqualTo(decodedLength);
+ assertThat(iut.chunks).isEqualTo(chunks);
+ }
+
+ private static Stream algorithms() {
+ return Arrays.stream(Algorithm.values());
+ }
+}
diff --git a/server/src/test/java/com/adobe/testing/s3mock/util/AwsChunkedInputStreamTest.java b/server/src/test/java/com/adobe/testing/s3mock/util/AwsUnsignedChunkedDecodingChecksumInputStreamTest.java
similarity index 67%
rename from server/src/test/java/com/adobe/testing/s3mock/util/AwsChunkedInputStreamTest.java
rename to server/src/test/java/com/adobe/testing/s3mock/util/AwsUnsignedChunkedDecodingChecksumInputStreamTest.java
index 78604c290..78eb1103f 100644
--- a/server/src/test/java/com/adobe/testing/s3mock/util/AwsChunkedInputStreamTest.java
+++ b/server/src/test/java/com/adobe/testing/s3mock/util/AwsUnsignedChunkedDecodingChecksumInputStreamTest.java
@@ -16,6 +16,8 @@
package com.adobe.testing.s3mock.util;
+import static com.adobe.testing.s3mock.dto.ChecksumAlgorithm.SHA256;
+import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_SHA256;
import static com.adobe.testing.s3mock.util.DigestUtil.checksumFor;
import static com.adobe.testing.s3mock.util.TestUtil.getFileFromClasspath;
import static java.nio.file.Files.newInputStream;
@@ -25,87 +27,71 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.Arrays;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
-import software.amazon.awssdk.auth.signer.internal.chunkedencoding.AwsS3V4ChunkSigner;
-import software.amazon.awssdk.auth.signer.internal.chunkedencoding.AwsSignedChunkedEncodingInputStream;
import software.amazon.awssdk.core.checksums.Algorithm;
import software.amazon.awssdk.core.checksums.SdkChecksum;
import software.amazon.awssdk.core.internal.chunked.AwsChunkedEncodingConfig;
import software.amazon.awssdk.core.internal.io.AwsUnsignedChunkedEncodingInputStream;
-class AwsChunkedInputStreamTest {
+class AwsUnsignedChunkedDecodingChecksumInputStreamTest {
- @ParameterizedTest
- @MethodSource("algorithms")
- void testDecode_unsigned_checksum(Algorithm algorithm, TestInfo testInfo) throws IOException {
- ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.fromString(algorithm.toString());
- String header = HeaderUtil.mapChecksumToHeader(checksumAlgorithm);
- doTestUnsigned(getFileFromClasspath(testInfo, "sampleFile.txt"),
- 1,
- header,
- SdkChecksum.forAlgorithm(algorithm),
- checksumFor(
- getFileFromClasspath(testInfo, "sampleFile.txt").toPath(), algorithm),
- checksumAlgorithm
- );
- doTestUnsigned(getFileFromClasspath(testInfo, "sampleFile_large.txt"),
- 16,
- header,
- SdkChecksum.forAlgorithm(algorithm),
- checksumFor(
- getFileFromClasspath(testInfo, "sampleFile_large.txt").toPath(), algorithm),
- checksumAlgorithm
- );
+ @Test
+ void testDecode_checksum(TestInfo testInfo) throws IOException {
+ doTest(testInfo, "sampleFile.txt", X_AMZ_CHECKSUM_SHA256, Algorithm.SHA256,
+ "1VcEifAruhjVvjzul4sC0B1EmlUdzqvsp6BP0KSVdTE=", SHA256);
+ doTest(testInfo, "sampleFile_large.txt", X_AMZ_CHECKSUM_SHA256, Algorithm.SHA256,
+ "Y8S4/uAGut7vjdFZQjLKZ7P28V9EPWb4BIoeniuM0mY=", SHA256);
}
@Test
- void testDecode_unsigned_noChecksum(TestInfo testInfo) throws IOException {
- doTestUnsigned(getFileFromClasspath(testInfo, "sampleFile.txt"), 1);
- doTestUnsigned(getFileFromClasspath(testInfo, "sampleFile_large.txt"), 16);
+ void testDecode_noChecksum(TestInfo testInfo) throws IOException {
+ doTest(testInfo, "sampleFile.txt");
+ doTest(testInfo, "sampleFile_large.txt");
}
- void doTestUnsigned(File input, int chunks) throws IOException {
- doTestUnsigned(input, chunks, null, null, null, null);
+ void doTest(TestInfo testInfo, String fileName) throws IOException {
+ doTest(testInfo, fileName, null, null, null, null);
}
- void doTestUnsigned(File input, int chunks, String header, SdkChecksum algorithm,
- String checksum, ChecksumAlgorithm checksumAlgorithm) throws IOException {
- InputStream chunkedEncodingInputStream = AwsUnsignedChunkedEncodingInputStream
+ void doTest(TestInfo testInfo, String fileName, String header, Algorithm algorithm,
+ String checksum, ChecksumAlgorithm checksumAlgorithm) throws IOException {
+ File sampleFile = getFileFromClasspath(testInfo, fileName);
+ AwsUnsignedChunkedEncodingInputStream.Builder builder = AwsUnsignedChunkedEncodingInputStream
.builder()
- .inputStream(newInputStream(input.toPath()))
- .sdkChecksum(algorithm)
+ .inputStream(Files.newInputStream(sampleFile.toPath()));
+ if (algorithm != null) {
+ builder.sdkChecksum(SdkChecksum.forAlgorithm(algorithm));
+ }
+ InputStream chunkedEncodingInputStream = builder
.checksumHeaderForTrailer(header)
//force chunks in the inputstream
.awsChunkedEncodingConfig(AwsChunkedEncodingConfig.builder().chunkSize(4000).build())
.build();
- long decodedLength = input.length();
- AwsChunkedInputStream iut = new AwsChunkedInputStream(chunkedEncodingInputStream,
- decodedLength,
- checksumAlgorithm != null);
+ long decodedLength = sampleFile.length();
+ AwsUnsignedChunkedDecodingChecksumInputStream iut = new
+ AwsUnsignedChunkedDecodingChecksumInputStream(chunkedEncodingInputStream, decodedLength);
- assertThat(iut).hasSameContentAs(newInputStream(input.toPath()));
+ assertThat(iut).hasSameContentAs(Files.newInputStream(sampleFile.toPath()));
assertThat(iut.getAlgorithm()).isEqualTo(checksumAlgorithm);
assertThat(iut.getChecksum()).isEqualTo(checksum);
assertThat(iut.decodedLength).isEqualTo(decodedLength);
assertThat(iut.readDecodedLength).isEqualTo(decodedLength);
- assertThat(iut.chunks).isEqualTo(chunks);
}
+
@ParameterizedTest
@MethodSource("algorithms")
- void testDecode_signed_checksum(Algorithm algorithm, TestInfo testInfo) throws IOException {
+ void testDecode_unsigned_checksum(Algorithm algorithm, TestInfo testInfo) throws IOException {
ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.fromString(algorithm.toString());
String header = HeaderUtil.mapChecksumToHeader(checksumAlgorithm);
- doTestSigned(getFileFromClasspath(testInfo, "sampleFile.txt"),
+ doTestUnsigned(getFileFromClasspath(testInfo, "sampleFile.txt"),
1,
header,
SdkChecksum.forAlgorithm(algorithm),
@@ -113,7 +99,7 @@ void testDecode_signed_checksum(Algorithm algorithm, TestInfo testInfo) throws I
getFileFromClasspath(testInfo, "sampleFile.txt").toPath(), algorithm),
checksumAlgorithm
);
- doTestSigned(getFileFromClasspath(testInfo, "sampleFile_large.txt"),
+ doTestUnsigned(getFileFromClasspath(testInfo, "sampleFile_large.txt"),
16,
header,
SdkChecksum.forAlgorithm(algorithm),
@@ -121,35 +107,51 @@ void testDecode_signed_checksum(Algorithm algorithm, TestInfo testInfo) throws I
getFileFromClasspath(testInfo, "sampleFile_large.txt").toPath(), algorithm),
checksumAlgorithm
);
+ doTestUnsigned(getFileFromClasspath(testInfo, "test-image-small.png"),
+ 9,
+ header,
+ SdkChecksum.forAlgorithm(algorithm),
+ checksumFor(
+ getFileFromClasspath(testInfo, "test-image-small.png").toPath(), algorithm),
+ checksumAlgorithm
+ );
+ doTestUnsigned(getFileFromClasspath(testInfo, "test-image.png"),
+ 17,
+ header,
+ SdkChecksum.forAlgorithm(algorithm),
+ checksumFor(
+ getFileFromClasspath(testInfo, "test-image.png").toPath(), algorithm),
+ checksumAlgorithm
+ );
}
@Test
- void testDecode_signed_noChecksum(TestInfo testInfo) throws IOException {
- doTestSigned(getFileFromClasspath(testInfo, "sampleFile.txt"), 1);
- doTestSigned(getFileFromClasspath(testInfo, "sampleFile_large.txt"), 16);
+ void testDecode_unsigned_noChecksum(TestInfo testInfo) throws IOException {
+ doTestUnsigned(getFileFromClasspath(testInfo, "sampleFile.txt"), 1);
+ doTestUnsigned(getFileFromClasspath(testInfo, "sampleFile_large.txt"), 16);
+ doTestUnsigned(getFileFromClasspath(testInfo, "test-image-small.png"), 9);
+ doTestUnsigned(getFileFromClasspath(testInfo, "test-image.png"), 17);
}
- void doTestSigned(File input, int chunks) throws IOException {
- doTestSigned(input, chunks, null, null, null, null);
+ void doTestUnsigned(File input, int chunks) throws IOException {
+ doTestUnsigned(input, chunks, null, null, null, null);
}
- void doTestSigned(File input, int chunks, String header, SdkChecksum algorithm,
- String checksum, ChecksumAlgorithm checksumAlgorithm) throws IOException {
- InputStream chunkedEncodingInputStream = AwsSignedChunkedEncodingInputStream
+ void doTestUnsigned(File input, int chunks, String header, SdkChecksum algorithm,
+ String checksum, ChecksumAlgorithm checksumAlgorithm) throws IOException {
+ InputStream chunkedEncodingInputStream = AwsUnsignedChunkedEncodingInputStream
.builder()
.inputStream(newInputStream(input.toPath()))
.sdkChecksum(algorithm)
.checksumHeaderForTrailer(header)
//force chunks in the inputstream
.awsChunkedEncodingConfig(AwsChunkedEncodingConfig.builder().chunkSize(4000).build())
- .awsChunkSigner(new AwsS3V4ChunkSigner("signingKey".getBytes(),
- "dateTime",
- "keyPath"))
.build();
long decodedLength = input.length();
- AwsChunkedInputStream iut = new AwsChunkedInputStream(chunkedEncodingInputStream, decodedLength,
- checksumAlgorithm != null);
+ AwsUnsignedChunkedDecodingChecksumInputStream iut =
+ new AwsUnsignedChunkedDecodingChecksumInputStream(chunkedEncodingInputStream,
+ decodedLength);
assertThat(iut).hasSameContentAs(newInputStream(input.toPath()));
assertThat(iut.getAlgorithm()).isEqualTo(checksumAlgorithm);
diff --git a/server/src/test/java/com/adobe/testing/s3mock/util/DigestUtilTest.java b/server/src/test/java/com/adobe/testing/s3mock/util/DigestUtilTest.java
index d3bf5e6e6..071de8c1f 100644
--- a/server/src/test/java/com/adobe/testing/s3mock/util/DigestUtilTest.java
+++ b/server/src/test/java/com/adobe/testing/s3mock/util/DigestUtilTest.java
@@ -1,5 +1,5 @@
/*
- * Copyright 2017-2023 Adobe.
+ * Copyright 2017-2024 Adobe.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,7 +19,6 @@
import static com.adobe.testing.s3mock.util.TestUtil.getTestFile;
import static org.assertj.core.api.Assertions.assertThat;
-import java.io.IOException;
import java.util.Arrays;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.ArrayUtils;
@@ -29,7 +28,7 @@
class DigestUtilTest {
@Test
- void testHexDigestOfMultipleFiles(TestInfo testInfo) throws IOException {
+ void testHexDigestOfMultipleFiles(TestInfo testInfo) {
//yes, this is correct - AWS calculates a Multipart digest by calculating the digest of every
//file involved, and then calculates the digest on the result.
//a hyphen with the part count is added as a suffix.
diff --git a/server/src/test/resources/test-image-small.png b/server/src/test/resources/test-image-small.png
new file mode 100644
index 000000000..348c20bd9
Binary files /dev/null and b/server/src/test/resources/test-image-small.png differ
diff --git a/server/src/test/resources/test-image.png b/server/src/test/resources/test-image.png
new file mode 100644
index 000000000..ec0f7b652
Binary files /dev/null and b/server/src/test/resources/test-image.png differ
diff --git a/testsupport/common/pom.xml b/testsupport/common/pom.xml
index 4416fa550..dba488fe8 100644
--- a/testsupport/common/pom.xml
+++ b/testsupport/common/pom.xml
@@ -22,7 +22,7 @@
com.adobe.testing
s3mock-testsupport-reactor
- 3.7.4-SNAPSHOT
+ 3.8.0-SNAPSHOT
s3mock-testsupport-common
diff --git a/testsupport/junit4/pom.xml b/testsupport/junit4/pom.xml
index 48320c5d4..61ce9adcf 100644
--- a/testsupport/junit4/pom.xml
+++ b/testsupport/junit4/pom.xml
@@ -22,7 +22,7 @@
com.adobe.testing
s3mock-testsupport-reactor
- 3.7.4-SNAPSHOT
+ 3.8.0-SNAPSHOT
s3mock-junit4
diff --git a/testsupport/junit5/pom.xml b/testsupport/junit5/pom.xml
index f7091402e..121477056 100644
--- a/testsupport/junit5/pom.xml
+++ b/testsupport/junit5/pom.xml
@@ -22,7 +22,7 @@
com.adobe.testing
s3mock-testsupport-reactor
- 3.7.4-SNAPSHOT
+ 3.8.0-SNAPSHOT
s3mock-junit5
diff --git a/testsupport/pom.xml b/testsupport/pom.xml
index 13d2dc546..c135fa3d0 100644
--- a/testsupport/pom.xml
+++ b/testsupport/pom.xml
@@ -22,7 +22,7 @@
com.adobe.testing
s3mock-parent
- 3.7.4-SNAPSHOT
+ 3.8.0-SNAPSHOT
s3mock-testsupport-reactor
diff --git a/testsupport/testcontainers/pom.xml b/testsupport/testcontainers/pom.xml
index d19d1d874..351b11d6f 100644
--- a/testsupport/testcontainers/pom.xml
+++ b/testsupport/testcontainers/pom.xml
@@ -22,7 +22,7 @@
com.adobe.testing
s3mock-testsupport-reactor
- 3.7.4-SNAPSHOT
+ 3.8.0-SNAPSHOT
s3mock-testcontainers
diff --git a/testsupport/testng/pom.xml b/testsupport/testng/pom.xml
index 435d7ac29..62e5a8dfd 100644
--- a/testsupport/testng/pom.xml
+++ b/testsupport/testng/pom.xml
@@ -22,7 +22,7 @@
com.adobe.testing
s3mock-testsupport-reactor
- 3.7.4-SNAPSHOT
+ 3.8.0-SNAPSHOT
s3mock-testng