Merge pull request #1890 from adobe/202405-test-its-against-s3

Test ITs against S3, fix tests and mock
adobe · May 26, 2024 · 0488cc1 · 0488cc1
2 parents 974c6b3 + 18d389c
commit 0488cc1
Show file tree

Hide file tree

Showing 63 changed files with 814 additions and 405 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,7 +12,7 @@
     * [Planned changes](#planned-changes)
 * [CURRENT - 3.x - THIS VERSION IS UNDER ACTIVE DEVELOPMENT](#current---3x---this-version-is-under-active-development)
   * [3.10.0 - PLANNED](#3100---planned)
-  * [3.9.0 - PLANNED](#390---planned)
+  * [3.9.0](#390)
   * [3.8.0](#380)
   * [3.7.3](#373)
   * [3.7.2](#372)
@@ -131,6 +131,7 @@ Version 3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
 * Features and fixes
   * Support Versions in APIs
   * Add "DeleteObjectTagging" API
+  * Support "Ownership" in buckets. ACLs should be 
 * Refactorings
   * TBD
 * Version updates
@@ -142,13 +143,22 @@ Version 3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
 * Features and fixes
   * Persist metadata for parts, validate checksum on multipart completion (fixes #1205)
 * Refactorings
-  * Migrate Unit tests to Kotlin 
+  * Migrate Unit tests to Kotlin
+  * Run ITs against real S3, fix code or tests in case of errors
+    * Fix Checksums for Multiparts
+    * Add ObjectOwnership config for Buckets, setting ACLs is not allowed otherwise
+    * Fix StorageClass, it's not returned for most APIs if it's "STANDARD"
 * Version updates
-  * Bump aws-v2.version from 2.25.49 to 2.25.54
-  * Bump com.amazonaws:aws-java-sdk-s3 from 1.12.720 to 1.12.724
-  * Bump actions/checkout from 4.1.5 to 4.1.6
-  * Bump github/codeql-action from 3.25.4 to 3.25.5
+  * Bump aws-v2.version from 2.25.49 to 2.25.59
+  * Bump com.amazonaws:aws-java-sdk-s3 from 1.12.720 to 1.12.729
+  * Bump kotlin.version from 1.9.24 to 2.0.0
+  * Bump alpine from 3.19.1 to 3.20.0 in /docker
+  * Bump org.codehaus.mojo:exec-maven-plugin from 3.2.0 to 3.3.0
+  * Bump com.github.ekryd.sortpom:sortpom-maven-plugin from 3.4.1 to 4.0.0
   * Bump license-maven-plugin-git.version from 4.4 to 4.5
+  * Bump actions/checkout from 4.1.5 to 4.1.6
+  * Bump github/codeql-action from 3.25.4 to 3.25.6
+  * Bump step-security/harden-runner from 2.7.1 to 2.8.0
 
 ## 3.8.0
 3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.

diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
@@ -47,6 +47,11 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-core</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk-s3</artifactId>

diff --git a/...lin/com/adobe/testing/s3mock/its/AclIT.kt → ...n/com/adobe/testing/s3mock/its/AclITV2.kt b/...lin/com/adobe/testing/s3mock/its/AclIT.kt → ...n/com/adobe/testing/s3mock/its/AclITV2.kt
@@ -22,22 +22,35 @@ import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.TestInfo
 import software.amazon.awssdk.services.s3.S3Client
 import software.amazon.awssdk.services.s3.model.AccessControlPolicy
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest
 import software.amazon.awssdk.services.s3.model.GetObjectAclRequest
 import software.amazon.awssdk.services.s3.model.Grant
 import software.amazon.awssdk.services.s3.model.Grantee
 import software.amazon.awssdk.services.s3.model.ObjectCannedACL
+import software.amazon.awssdk.services.s3.model.ObjectOwnership
 import software.amazon.awssdk.services.s3.model.Owner
 import software.amazon.awssdk.services.s3.model.Permission.FULL_CONTROL
 import software.amazon.awssdk.services.s3.model.PutObjectAclRequest
 import software.amazon.awssdk.services.s3.model.Type.CANONICAL_USER
 
-internal class AclIT : S3TestBase() {
+internal class AclITV2 : S3TestBase() {
   private val s3ClientV2: S3Client = createS3ClientV2()
 
   @Test
+  @S3VerifiedSuccess(year = 2024)
   fun testPutCannedAcl_OK(testInfo: TestInfo) {
     val sourceKey = UPLOAD_FILE_NAME
-    val (bucketName, _) = givenBucketAndObjectV2(testInfo, sourceKey)
+    val bucketName = bucketName(testInfo)
+
+    //create bucket that sets ownership to non-default to allow setting ACLs.
+    s3ClientV2.createBucket(CreateBucketRequest
+      .builder()
+      .bucket(bucketName)
+      .objectOwnership(ObjectOwnership.OBJECT_WRITER)
+      .build()
+    )
+
+    givenObjectV2(bucketName, sourceKey)
 
     s3ClientV2.putObjectAcl(
       PutObjectAclRequest
@@ -58,8 +71,8 @@ internal class AclIT : S3TestBase() {
         .build()
     ).also {
       assertThat(it.sdkHttpResponse().isSuccessful).isTrue()
-      assertThat(it.owner().id()).isEqualTo(DEFAULT_OWNER.id)
-      assertThat(it.owner().displayName()).isEqualTo(DEFAULT_OWNER.displayName)
+      assertThat(it.owner().id()).isNotBlank()
+      assertThat(it.owner().displayName()).isNotBlank()
       assertThat(it.grants().size).isEqualTo(1)
       assertThat(it.grants()[0].permission()).isEqualTo(FULL_CONTROL)
     }

diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV1IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV1IT.kt
@@ -75,7 +75,7 @@ internal class BucketV1IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun testCreateAndDeleteBucket(testInfo: TestInfo) {
     val bucketName = bucketName(testInfo)
     s3Client.createBucket(bucketName)
@@ -88,7 +88,7 @@ internal class BucketV1IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun testFailureDeleteNonEmptyBucket(testInfo: TestInfo) {
     val bucketName = bucketName(testInfo)
     s3Client.createBucket(bucketName)
@@ -101,7 +101,7 @@ internal class BucketV1IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun testBucketDoesExistV2_ok(testInfo: TestInfo) {
     val bucketName = bucketName(testInfo)
     s3Client.createBucket(bucketName)
@@ -112,7 +112,7 @@ internal class BucketV1IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun testBucketDoesExistV2_failure(testInfo: TestInfo) {
     val bucketName = bucketName(testInfo)
 
@@ -121,7 +121,7 @@ internal class BucketV1IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun duplicateBucketCreation(testInfo: TestInfo) {
     val bucketName = bucketName(testInfo)
     s3Client.createBucket(bucketName)
@@ -135,7 +135,7 @@ internal class BucketV1IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun duplicateBucketDeletion(testInfo: TestInfo) {
     val bucketName = bucketName(testInfo)
     s3Client.createBucket(bucketName)

diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV2IT.kt
@@ -48,7 +48,7 @@ internal class BucketV2IT : S3TestBase() {
   private val s3ClientV2: S3Client = createS3ClientV2()
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun createAndDeleteBucket(testInfo: TestInfo) {
     val bucketName = bucketName(testInfo)
     s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build())
@@ -71,7 +71,7 @@ internal class BucketV2IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun getBucketLocation(testInfo: TestInfo) {
     val bucketName = givenBucketV2(testInfo)
     val bucketLocation = s3ClientV2.getBucketLocation(GetBucketLocationRequest.builder().bucket(bucketName).build())
@@ -80,7 +80,7 @@ internal class BucketV2IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun duplicateBucketCreation(testInfo: TestInfo) {
     val bucketName = bucketName(testInfo)
     s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build())
@@ -112,7 +112,7 @@ internal class BucketV2IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun duplicateBucketDeletion(testInfo: TestInfo) {
     val bucketName = bucketName(testInfo)
     s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build())
@@ -143,7 +143,7 @@ internal class BucketV2IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun getBucketLifecycle_notFound(testInfo: TestInfo) {
     val bucketName = bucketName(testInfo)
     s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build())
@@ -167,7 +167,7 @@ internal class BucketV2IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun putGetDeleteBucketLifecycle(testInfo: TestInfo) {
     val bucketName = bucketName(testInfo)
     s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build())

diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CopyObjectV1IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CopyObjectV1IT.kt
@@ -47,7 +47,7 @@ internal class CopyObjectV1IT : S3TestBase() {
    * compares checksums of original and copied object.
    */
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun shouldCopyObject(testInfo: TestInfo) {
     val sourceKey = UPLOAD_FILE_NAME
     val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, sourceKey)
@@ -65,7 +65,7 @@ internal class CopyObjectV1IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun testCopyObject_successMatch(testInfo: TestInfo) {
     val sourceKey = UPLOAD_FILE_NAME
     val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, sourceKey)
@@ -85,7 +85,7 @@ internal class CopyObjectV1IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun testCopyObject_successNoneMatch(testInfo: TestInfo) {
     val sourceKey = UPLOAD_FILE_NAME
     val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, sourceKey)
@@ -105,7 +105,7 @@ internal class CopyObjectV1IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun testCopyObject_failureMatch(testInfo: TestInfo) {
     val sourceKey = UPLOAD_FILE_NAME
     val (bucketName, _) = givenBucketAndObjectV1(testInfo, sourceKey)
@@ -127,7 +127,7 @@ internal class CopyObjectV1IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun testCopyObject_failureNoneMatch(testInfo: TestInfo) {
     val sourceKey = UPLOAD_FILE_NAME
     val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, sourceKey)
@@ -153,7 +153,7 @@ internal class CopyObjectV1IT : S3TestBase() {
    * Downloads the object; compares checksums of original and copied object.
    */
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun shouldCopyObjectToSameKey(testInfo: TestInfo) {
     val bucketName = givenBucketV1(testInfo)
     val uploadFile = File(UPLOAD_FILE_NAME)
@@ -164,16 +164,19 @@ internal class CopyObjectV1IT : S3TestBase() {
     val putObjectResult = PutObjectRequest(bucketName, sourceKey, uploadFile).withMetadata(objectMetadata).let {
       s3Client.putObject(it)
     }
-    //TODO: this is actually illegal on S3. when copying to the same key like this, S3 will throw:
-    // This copy request is illegal because it is trying to copy an object to itself without
-    // changing the object's metadata, storage class, website redirect location or encryption attributes.
-    CopyObjectRequest(bucketName, sourceKey, bucketName, sourceKey).also {
+
+    CopyObjectRequest(bucketName, sourceKey, bucketName, sourceKey).apply {
+     this.newObjectMetadata = ObjectMetadata().apply {
+       this.userMetadata = mapOf("test-key1" to "test-value1")
+     }
+    }.also {
       s3Client.copyObject(it)
     }
 
     s3Client.getObject(bucketName, sourceKey).use {
       val copiedObjectMetadata = it.objectMetadata
-      assertThat(copiedObjectMetadata.userMetadata["test-key"]).isEqualTo("test-value")
+      assertThat(copiedObjectMetadata.userMetadata["test-key"]).isNull()
+      assertThat(copiedObjectMetadata.userMetadata["test-key1"]).isEqualTo("test-value1")
 
       val objectContent = it.objectContent
       val copiedDigest = DigestUtil.hexDigest(objectContent)
@@ -186,7 +189,7 @@ internal class CopyObjectV1IT : S3TestBase() {
    * Downloads the object; compares checksums of original and copied object.
    */
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun shouldCopyObjectWithReplaceToSameKey(testInfo: TestInfo) {
     val bucketName = givenBucketV1(testInfo)
     val uploadFile = File(UPLOAD_FILE_NAME)
@@ -231,7 +234,7 @@ internal class CopyObjectV1IT : S3TestBase() {
    * the new user metadata specified during copy request.
    */
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun shouldCopyObjectWithNewUserMetadata(testInfo: TestInfo) {
     val sourceKey = UPLOAD_FILE_NAME
     val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, sourceKey)
@@ -261,7 +264,7 @@ internal class CopyObjectV1IT : S3TestBase() {
    * the source object user metadata;
    */
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun shouldCopyObjectWithSourceUserMetadata(testInfo: TestInfo) {
     val bucketName = givenBucketV1(testInfo)
     val uploadFile = File(UPLOAD_FILE_NAME)
@@ -292,7 +295,7 @@ internal class CopyObjectV1IT : S3TestBase() {
    * @see .shouldCopyObject
    */
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun shouldCopyObjectToKeyNeedingEscaping(testInfo: TestInfo) {
     val bucketName = givenBucketV1(testInfo)
     val uploadFile = File(UPLOAD_FILE_NAME)
@@ -316,7 +319,7 @@ internal class CopyObjectV1IT : S3TestBase() {
    * @see .shouldCopyObject
    */
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun shouldCopyObjectFromKeyNeedingEscaping(testInfo: TestInfo) {
     val bucketName = givenBucketV1(testInfo)
     val uploadFile = File(UPLOAD_FILE_NAME)
@@ -368,7 +371,7 @@ internal class CopyObjectV1IT : S3TestBase() {
    * Tests that an object won't be copied with wrong encryption Key.
    */
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun shouldNotObjectCopyWithWrongEncryptionKey(testInfo: TestInfo) {
     val sourceKey = UPLOAD_FILE_NAME
     val (bucketName, _) = givenBucketAndObjectV1(testInfo, sourceKey)
@@ -387,7 +390,7 @@ internal class CopyObjectV1IT : S3TestBase() {
    * Tests that a copy request for a non-existing object throws the correct error.
    */
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun shouldThrowNoSuchKeyOnCopyForNonExistingKey(testInfo: TestInfo) {
     val bucketName = givenBucketV1(testInfo)
     val sourceKey = randomName
@@ -401,7 +404,7 @@ internal class CopyObjectV1IT : S3TestBase() {
   }
 
   @Test
-  @S3VerifiedSuccess(year = 2022)
+  @S3VerifiedSuccess(year = 2024)
   fun multipartCopy() {
     //content larger than default part threshold of 5MiB
     val contentLen = 10 * _1MB