Merge remote-tracking branch 'upstream/main' into 5962-audit-cli-comm…

…and-model
apache · Jan 2, 2025 · 37ce144 · 37ce144
2 parents 8ce2946 + ece06fa
commit 37ce144
Show file tree

Hide file tree

Showing 16 changed files with 989 additions and 94 deletions.
diff --git a/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java
@@ -138,9 +138,10 @@ private String createPolicy(Set<String> readLocations, Set<String> writeLocation
             .effect(Effect.ALLOW)
             .addAction("oss:GetObject")
             .addAction("oss:GetObjectVersion");
+
     // Add support for bucket-level policies
     Map<String, Statement.Builder> bucketListStatementBuilder = new HashMap<>();
-    Map<String, Statement.Builder> bucketGetLocationStatementBuilder = new HashMap<>();
+    Map<String, Statement.Builder> bucketMetadataStatementBuilder = new HashMap<>();
 
     String arnPrefix = getArnPrefix();
     Stream.concat(readLocations.stream(), writeLocations.stream())
@@ -150,22 +151,24 @@ private String createPolicy(Set<String> readLocations, Set<String> writeLocation
               URI uri = URI.create(location);
               allowGetObjectStatementBuilder.addResource(getOssUriWithArn(arnPrefix, uri));
               String bucketArn = arnPrefix + getBucketName(uri);
-              // ListBucket
+              // OSS use 'oss:ListObjects' to list objects in a bucket while s3 use 's3:ListBucket'
               bucketListStatementBuilder.computeIfAbsent(
                   bucketArn,
                   key ->
                       Statement.builder()
                           .effect(Effect.ALLOW)
-                          .addAction("oss:ListBucket")
+                          .addAction("oss:ListObjects")
                           .addResource(key)
                           .condition(getCondition(uri)));
-              // GetBucketLocation
-              bucketGetLocationStatementBuilder.computeIfAbsent(
+              // Add get bucket location and bucket info action.
+              bucketMetadataStatementBuilder.computeIfAbsent(
                   bucketArn,
                   key ->
                       Statement.builder()
                           .effect(Effect.ALLOW)
                           .addAction("oss:GetBucketLocation")
+                          // Required for OSS Hadoop connector to get bucket information
+                          .addAction("oss:GetBucketInfo")
                           .addResource(key));
             });
 
@@ -192,7 +195,7 @@ private String createPolicy(Set<String> readLocations, Set<String> writeLocation
       policyBuilder.addStatement(
           Statement.builder().effect(Effect.ALLOW).addAction("oss:ListBucket").build());
     }
-    bucketGetLocationStatementBuilder
+    bucketMetadataStatementBuilder
         .values()
         .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build()));
 

diff --git a/bundles/aws/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java b/bundles/aws/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java
@@ -20,6 +20,7 @@
 package org.apache.gravitino.s3.credential;
 
 import java.net.URI;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -49,6 +50,7 @@
 
 /** Generates S3 token to access S3 data. */
 public class S3TokenProvider implements CredentialProvider {
+
   private StsClient stsClient;
   private String roleArn;
   private String externalID;
@@ -134,6 +136,7 @@ private IamPolicy createPolicy(
               allowGetObjectStatementBuilder.addResource(
                   IamResource.create(getS3UriWithArn(arnPrefix, uri)));
               String bucketArn = arnPrefix + getBucketName(uri);
+              String rawPath = trimLeadingSlash(uri.getPath());
               bucketListStatmentBuilder
                   .computeIfAbsent(
                       bucketArn,
@@ -142,10 +145,14 @@ private IamPolicy createPolicy(
                               .effect(IamEffect.ALLOW)
                               .addAction("s3:ListBucket")
                               .addResource(key))
-                  .addCondition(
+                  .addConditions(
                       IamConditionOperator.STRING_LIKE,
                       "s3:prefix",
-                      concatPathWithSep(trimLeadingSlash(uri.getPath()), "*", "/"));
+                      Arrays.asList(
+                          // Get raw path metadata information for AWS hadoop connector
+                          rawPath,
+                          // Listing objects in raw path
+                          concatPathWithSep(rawPath, "*", "/")));
               bucketGetLocationStatmentBuilder.computeIfAbsent(
                   bucketArn,
                   key ->

diff --git a/catalogs/catalog-model/build.gradle.kts b/catalogs/catalog-model/build.gradle.kts
@@ -29,32 +29,33 @@ dependencies {
     exclude(group = "*")
   }
 
-  implementation(project(":core")) {
+  implementation(project(":catalogs:catalog-common")) {
     exclude(group = "*")
   }
   implementation(project(":common")) {
     exclude(group = "*")
   }
-
-  implementation(project(":catalogs:catalog-common")) {
+  implementation(project(":core")) {
     exclude(group = "*")
   }
-
   implementation(libs.guava)
   implementation(libs.slf4j.api)
 
   testImplementation(project(":clients:client-java"))
   testImplementation(project(":integration-test-common", "testArtifacts"))
   testImplementation(project(":server"))
   testImplementation(project(":server-common"))
-
   testImplementation(libs.bundles.log4j)
   testImplementation(libs.commons.io)
   testImplementation(libs.commons.lang3)
   testImplementation(libs.mockito.core)
   testImplementation(libs.mockito.inline)
+  testImplementation(libs.mysql.driver)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
+  testImplementation(libs.postgresql.driver)
+  testImplementation(libs.testcontainers)
+  testImplementation(libs.testcontainers.mysql)
 
   testRuntimeOnly(libs.junit.jupiter.engine)
 }
@@ -68,8 +69,9 @@ tasks {
   val copyCatalogLibs by registering(Copy::class) {
     dependsOn("jar", "runtimeJars")
     from("build/libs") {
-      exclude("slf4j-*.jar")
       exclude("guava-*.jar")
+      exclude("log4j-*.jar")
+      exclude("slf4j-*.jar")
     }
     into("$rootDir/distribution/package/catalogs/model/libs")
   }