Merge pull request #1828 from adobe/1827-validate-checksums-for-uploa…

…ds-and-parts Verify checksums on upload
adobe · May 1, 2024 · 48a64aa · 48a64aa
2 parents 434092e + 73ce586
commit 48a64aa
Show file tree

Hide file tree

Showing 29 changed files with 1,086 additions and 733 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,7 +12,9 @@
     * [Planned changes](#planned-changes)
 * [CURRENT - 3.x - THIS VERSION IS UNDER ACTIVE DEVELOPMENT](#current---3x---this-version-is-under-active-development)
   * [3.8.0 - PLANNED](#380---planned)
-  * [3.7.1 - PLANNED](#371---planned)
+  * [3.7.3 - PLANNED](#373---planned)
+  * [3.7.2](#372)
+  * [3.7.1](#371)
   * [3.7.0](#370)
   * [3.6.0](#360)
   * [3.5.2](#352)
@@ -132,15 +134,18 @@ Version 3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
 * Version updates
   * TBD
 
-## 3.7.2 - PLANNED
+## 3.7.3 - PLANNED
 3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
 
 * Features and fixes
-  * Calculate and validate checksums on upload
-* Refactorings
-  * TBD
-* Version updates
-  * TBD
+  * Support large, chunked, unsigned, asynchronous uploads (fixes #1818)
+
+## 3.7.2
+3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
+
+* Features and fixes
+  * Calculate and validate checksums on upload (fixes #1827)
+    * UploadPart API now also returns checksums, if available.
 
 ## 3.7.1
 3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.

diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AclIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AclIT.kt
@@ -39,28 +39,30 @@ internal class AclIT : S3TestBase() {
     val sourceKey = UPLOAD_FILE_NAME
     val (bucketName, _) = givenBucketAndObjectV2(testInfo, sourceKey)
 
-    val putAclResponse = s3ClientV2.putObjectAcl(
+    s3ClientV2.putObjectAcl(
       PutObjectAclRequest
         .builder()
         .bucket(bucketName)
         .key(sourceKey)
         .acl(ObjectCannedACL.PRIVATE)
         .build()
-    )
-    assertThat(putAclResponse.sdkHttpResponse().isSuccessful).isTrue()
+    ).also {
+      assertThat(it.sdkHttpResponse().isSuccessful).isTrue()
+    }
 
-    val getAclResponse = s3ClientV2.getObjectAcl(
+    s3ClientV2.getObjectAcl(
       GetObjectAclRequest
         .builder()
         .bucket(bucketName)
         .key(sourceKey)
         .build()
-    )
-    assertThat(getAclResponse.sdkHttpResponse().isSuccessful).isTrue()
-    assertThat(getAclResponse.owner().id()).isEqualTo(DEFAULT_OWNER.id)
-    assertThat(getAclResponse.owner().displayName()).isEqualTo(DEFAULT_OWNER.displayName)
-    assertThat(getAclResponse.grants().size).isEqualTo(1)
-    assertThat(getAclResponse.grants()[0].permission()).isEqualTo(FULL_CONTROL)
+    ).also {
+      assertThat(it.sdkHttpResponse().isSuccessful).isTrue()
+      assertThat(it.owner().id()).isEqualTo(DEFAULT_OWNER.id)
+      assertThat(it.owner().displayName()).isEqualTo(DEFAULT_OWNER.displayName)
+      assertThat(it.grants().size).isEqualTo(1)
+      assertThat(it.grants()[0].permission()).isEqualTo(FULL_CONTROL)
+    }
   }
 
   @Test
@@ -78,21 +80,22 @@ internal class AclIT : S3TestBase() {
         .build()
     )
 
-    val owner = acl.owner()
-    assertThat(owner.id()).isEqualTo(DEFAULT_OWNER.id)
-    assertThat(owner.displayName()).isEqualTo(DEFAULT_OWNER.displayName)
-
-    val grants = acl.grants()
-    assertThat(grants).hasSize(1)
+    acl.owner().also { owner ->
+      assertThat(owner.id()).isEqualTo(DEFAULT_OWNER.id)
+      assertThat(owner.displayName()).isEqualTo(DEFAULT_OWNER.displayName)
+    }
 
+    val grants = acl.grants().also {
+      assertThat(it).hasSize(1)
+    }
     val grant = grants[0]
     assertThat(grant.permission()).isEqualTo(FULL_CONTROL)
-
-    val grantee = grant.grantee()
-    assertThat(grantee).isNotNull
-    assertThat(grantee.id()).isEqualTo(DEFAULT_OWNER.id)
-    assertThat(grantee.displayName()).isEqualTo(DEFAULT_OWNER.displayName)
-    assertThat(grantee.type()).isEqualTo(CANONICAL_USER)
+    grant.grantee().also {
+      assertThat(it).isNotNull
+      assertThat(it.id()).isEqualTo(DEFAULT_OWNER.id)
+      assertThat(it.displayName()).isEqualTo(DEFAULT_OWNER.displayName)
+      assertThat(it.type()).isEqualTo(CANONICAL_USER)
+    }
   }
 
 

diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingITV2.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingITV2.kt
@@ -62,22 +62,25 @@ internal class AwsChunkedEndcodingITV2 : S3TestBase() {
       RequestBody.fromFile(uploadFile)
     )
 
-    val putChecksum = putObjectResponse.checksumSHA256()
-    assertThat(putChecksum).isNotBlank
-    assertThat(putChecksum).isEqualTo(expectedChecksum)
+    putObjectResponse.checksumSHA256().also {
+      assertThat(it).isNotBlank
+      assertThat(it).isEqualTo(expectedChecksum)
+    }
 
-    val getObjectResponse = s3ClientV2.getObject(
+    s3ClientV2.getObject(
       GetObjectRequest.builder()
         .bucket(bucket)
         .key(UPLOAD_FILE_NAME)
         .build()
-    )
-    assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag)
-    assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length())
+    ).also { getObjectResponse ->
+      assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag)
+      assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length())
 
-    val getChecksum = getObjectResponse.response().checksumSHA256()
-    assertThat(getChecksum).isNotBlank
-    assertThat(getChecksum).isEqualTo(expectedChecksum)
+      getObjectResponse.response().checksumSHA256().also {
+        assertThat(it).isNotBlank
+        assertThat(it).isEqualTo(expectedChecksum)
+      }
+    }
   }
 
   /**
@@ -104,13 +107,14 @@ internal class AwsChunkedEndcodingITV2 : S3TestBase() {
       RequestBody.fromFile(uploadFile)
     )
 
-    val getObjectResponse = s3ClientV2.getObject(
+    s3ClientV2.getObject(
       GetObjectRequest.builder()
         .bucket(bucket)
         .key(UPLOAD_FILE_NAME)
         .build()
-    )
-    assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag)
-    assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length())
+    ).also {
+      assertThat(it.response().eTag()).isEqualTo(expectedEtag)
+      assertThat(it.response().contentLength()).isEqualTo(uploadFile.length())
+    }
   }
 }
diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV1IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV1IT.kt
@@ -55,9 +55,10 @@ internal class BucketV1IT : S3TestBase() {
     val createdBucket = buckets[0]
     assertThat(createdBucket.creationDate).isAfterOrEqualTo(creationDate)
 
-    val bucketOwner = createdBucket.owner
-    assertThat(bucketOwner.displayName).isEqualTo("s3-mock-file-store")
-    assertThat(bucketOwner.id).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
+    createdBucket.owner.also {
+      assertThat(it.displayName).isEqualTo("s3-mock-file-store")
+      assertThat(it.id).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
+    }
   }
 
   @Test
@@ -81,8 +82,9 @@ internal class BucketV1IT : S3TestBase() {
     s3Client.headBucket(HeadBucketRequest(bucketName))
     s3Client.deleteBucket(bucketName)
 
-    val doesBucketExist = s3Client.doesBucketExistV2(bucketName)
-    assertThat(doesBucketExist).isFalse
+    s3Client.doesBucketExistV2(bucketName).also {
+      assertThat(it).isFalse
+    }
   }
 
   @Test
@@ -104,8 +106,9 @@ internal class BucketV1IT : S3TestBase() {
     val bucketName = bucketName(testInfo)
     s3Client.createBucket(bucketName)
 
-    val doesBucketExist = s3Client.doesBucketExistV2(bucketName)
-    assertThat(doesBucketExist).isTrue
+    s3Client.doesBucketExistV2(bucketName).also {
+      assertThat(it).isTrue
+    }
   }
 
   @Test

diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV2IT.kt
@@ -64,9 +64,10 @@ internal class BucketV2IT : S3TestBase() {
     s3ClientV2.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build())
     val bucketDeleted = s3ClientV2.waiter()
       .waitUntilBucketNotExists(HeadBucketRequest.builder().bucket(bucketName).build())
-    val bucketDeletedResponse = bucketDeleted.matched().exception().get()
-    assertThat(bucketDeletedResponse).isNotNull
-    assertThat(bucketDeletedResponse).isInstanceOf(NoSuchBucketException::class.java)
+    bucketDeleted.matched().exception().get().also {
+      assertThat(it).isNotNull
+      assertThat(it).isInstanceOf(NoSuchBucketException::class.java)
+    }
   }
 
   @Test
@@ -86,8 +87,9 @@ internal class BucketV2IT : S3TestBase() {
 
     val bucketCreated = s3ClientV2.waiter()
       .waitUntilBucketExists(HeadBucketRequest.builder().bucket(bucketName).build())
-    val bucketCreatedResponse = bucketCreated.matched().response().get()
-    assertThat(bucketCreatedResponse).isNotNull
+    bucketCreated.matched().response().get().also {
+      assertThat(it).isNotNull
+    }
 
     assertThatThrownBy {
       s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build())
@@ -103,9 +105,10 @@ internal class BucketV2IT : S3TestBase() {
     val bucketDeleted = s3ClientV2.waiter()
       .waitUntilBucketNotExists(HeadBucketRequest.builder().bucket(bucketName).build())
 
-    val bucketDeletedResponse = bucketDeleted.matched().exception().get()
-    assertThat(bucketDeletedResponse).isNotNull
-    assertThat(bucketDeletedResponse).isInstanceOf(NoSuchBucketException::class.java)
+    bucketDeleted.matched().exception().get().also {
+      assertThat(it).isNotNull
+      assertThat(it).isInstanceOf(NoSuchBucketException::class.java)
+    }
   }
 
   @Test
@@ -116,15 +119,17 @@ internal class BucketV2IT : S3TestBase() {
 
     val bucketCreated = s3ClientV2.waiter()
       .waitUntilBucketExists(HeadBucketRequest.builder().bucket(bucketName).build())
-    val bucketCreatedResponse = bucketCreated.matched().response().get()
-    assertThat(bucketCreatedResponse).isNotNull
+    bucketCreated.matched().response().get().also {
+      assertThat(it).isNotNull
+    }
 
     s3ClientV2.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build())
     val bucketDeleted = s3ClientV2.waiter()
       .waitUntilBucketNotExists(HeadBucketRequest.builder().bucket(bucketName).build())
-    val bucketDeletedResponse = bucketDeleted.matched().exception().get()
-    assertThat(bucketDeletedResponse).isNotNull
-    assertThat(bucketDeletedResponse).isInstanceOf(NoSuchBucketException::class.java)
+    bucketDeleted.matched().exception().get().also {
+      assertThat(it).isNotNull
+      assertThat(it).isInstanceOf(NoSuchBucketException::class.java)
+    }
 
     assertThatThrownBy {
       s3ClientV2.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build())
@@ -169,8 +174,9 @@ internal class BucketV2IT : S3TestBase() {
 
     val bucketCreated = s3ClientV2.waiter()
       .waitUntilBucketExists(HeadBucketRequest.builder().bucket(bucketName).build())
-    val bucketCreatedResponse = bucketCreated.matched().response()!!.get()
-    assertThat(bucketCreatedResponse).isNotNull
+    bucketCreated.matched().response()!!.get().also {
+      assertThat(it).isNotNull
+    }
 
     val configuration = BucketLifecycleConfiguration
       .builder()
@@ -206,20 +212,21 @@ internal class BucketV2IT : S3TestBase() {
         .build()
     )
 
-    val configurationResponse = s3ClientV2.getBucketLifecycleConfiguration(
+    s3ClientV2.getBucketLifecycleConfiguration(
       GetBucketLifecycleConfigurationRequest
         .builder()
         .bucket(bucketName)
         .build()
-    )
-
-    assertThat(configurationResponse.rules()[0]).isEqualTo(configuration.rules()[0])
+    ).also {
+      assertThat(it.rules()[0]).isEqualTo(configuration.rules()[0])
+    }
 
-    val deleteBucketLifecycle = s3ClientV2.deleteBucketLifecycle(
+    s3ClientV2.deleteBucketLifecycle(
       DeleteBucketLifecycleRequest.builder().bucket(bucketName).build()
-    )
+    ).also {
+      assertThat(it.sdkHttpResponse().statusCode()).isEqualTo(204)
+    }
 
-    assertThat(deleteBucketLifecycle.sdkHttpResponse().statusCode()).isEqualTo(204)
 
     // give AWS time to actually delete the lifecycleConfiguration, otherwise the following call
     // will not fail as expected...

diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ConcurrencyIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ConcurrencyIT.kt
@@ -61,31 +61,35 @@ internal class ConcurrencyIT : S3TestBase() {
   inner class Runner(val bucketName: String, val key: String) : Callable<Boolean> {
     override fun call(): Boolean {
       LATCH.countDown()
-      val putObjectResponse = s3ClientV2.putObject(
+      s3ClientV2.putObject(
         PutObjectRequest
           .builder()
           .bucket(bucketName)
           .key(key)
           .build(), RequestBody.empty()
-      )
-      assertThat(putObjectResponse.eTag()).isNotBlank
-      val getObjectResponse = s3ClientV2.getObject(
+      ).also {
+        assertThat(it.eTag()).isNotBlank
+      }
+
+      s3ClientV2.getObject(
         GetObjectRequest
           .builder()
           .bucket(bucketName)
           .key(key)
           .build()
-      )
-      assertThat(getObjectResponse.response().eTag()).isNotBlank
+      ).also {
+        assertThat(it.response().eTag()).isNotBlank
+      }
 
-      val deleteObjectResponse = s3ClientV2.deleteObject(
+      s3ClientV2.deleteObject(
         DeleteObjectRequest
           .builder()
           .bucket(bucketName)
           .key(key)
           .build()
-      )
-      assertThat(deleteObjectResponse.deleteMarker()).isTrue
+      ).also {
+        assertThat(it.deleteMarker()).isTrue
+      }
       DONE.incrementAndGet()
       return true
     }