Merge branch 'main' of github.com:datastrato/graviton into 5472

authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationUserGroupMappingProvider.java

@@ -0,0 +1,64 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing,
+ *  software distributed under the License is distributed on an
+ *  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  KIND, either express or implied.  See the License for the
+ *  specific language governing permissions and limitations
+ *  under the License.
+ *
+ */
+
+package org.apache.gravitino.authorization.common;
+
+import java.util.Map;
+
+/**
+ * The AuthorizationUserGroupMappingProvider interface defines the public API for mapping Gravitino
+ * users and groups to the that in underlying data source system.
+ *
+ * <p>Typically, the users and group names in Gravitino are the same as the underlying data source.
+ * However, in some cases, the user and group names in Gravitino may be different from the
+ * underlying data source. For instance, in GCP IAM, the username is the email address or the
+ * service account. So the user group mapping provider can be used to map the Gravitino username to
+ * the email address or service account.
+ */
+public interface AuthorizationUserGroupMappingProvider {
+
+  /**
+   * Initialize the user group mapping provider with the configuration.
+   *
+   * @param config The configuration map for the user group mapping provider.
+   */
+  default void initialize(Map<String, String> config) {}
+
+  /**
+   * Get the username from the underlying data source based on the Gravitino username For instance,
+   * in GCP IAM, the username is the email address or the service account.
+   *
+   * @param gravitinoUserName The Gravitino username.
+   * @return The username from the underlying data source.
+   */
+  default String getUserName(String gravitinoUserName) {
+    return gravitinoUserName;
+  }
+
+  /**
+   * Get the group name from the underlying data source based on the Gravitino group name.
+   *
+   * @param gravitinoGroupName The Gravitino group name.
+   * @return The group name from the underlying data source.
+   */
+  default String getGroupName(String gravitinoGroupName) {
+    return gravitinoGroupName;
+  }
+}

bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java

@@ -138,9 +138,10 @@ private String createPolicy(Set<String> readLocations, Set<String> writeLocation
             .effect(Effect.ALLOW)
             .addAction("oss:GetObject")
             .addAction("oss:GetObjectVersion");
+
     // Add support for bucket-level policies
     Map<String, Statement.Builder> bucketListStatementBuilder = new HashMap<>();
-    Map<String, Statement.Builder> bucketGetLocationStatementBuilder = new HashMap<>();
+    Map<String, Statement.Builder> bucketMetadataStatementBuilder = new HashMap<>();
 
     String arnPrefix = getArnPrefix();
     Stream.concat(readLocations.stream(), writeLocations.stream())
@@ -150,22 +151,24 @@ private String createPolicy(Set<String> readLocations, Set<String> writeLocation
               URI uri = URI.create(location);
               allowGetObjectStatementBuilder.addResource(getOssUriWithArn(arnPrefix, uri));
               String bucketArn = arnPrefix + getBucketName(uri);
-              // ListBucket
+              // OSS use 'oss:ListObjects' to list objects in a bucket while s3 use 's3:ListBucket'
               bucketListStatementBuilder.computeIfAbsent(
                   bucketArn,
                   key ->
                       Statement.builder()
                           .effect(Effect.ALLOW)
-                          .addAction("oss:ListBucket")
+                          .addAction("oss:ListObjects")
                           .addResource(key)
                           .condition(getCondition(uri)));
-              // GetBucketLocation
-              bucketGetLocationStatementBuilder.computeIfAbsent(
+              // Add get bucket location and bucket info action.
+              bucketMetadataStatementBuilder.computeIfAbsent(
                   bucketArn,
                   key ->
                       Statement.builder()
                           .effect(Effect.ALLOW)
                           .addAction("oss:GetBucketLocation")
+                          // Required for OSS Hadoop connector to get bucket information
+                          .addAction("oss:GetBucketInfo")
                           .addResource(key));
             });
 
@@ -192,7 +195,7 @@ private String createPolicy(Set<String> readLocations, Set<String> writeLocation
       policyBuilder.addStatement(
           Statement.builder().effect(Effect.ALLOW).addAction("oss:ListBucket").build());
     }
-    bucketGetLocationStatementBuilder
+    bucketMetadataStatementBuilder
         .values()
         .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build()));
 

bundles/aws/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java

@@ -20,6 +20,7 @@
 package org.apache.gravitino.s3.credential;
 
 import java.net.URI;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -49,6 +50,7 @@
 
 /** Generates S3 token to access S3 data. */
 public class S3TokenProvider implements CredentialProvider {
+
   private StsClient stsClient;
   private String roleArn;
   private String externalID;
@@ -134,6 +136,7 @@ private IamPolicy createPolicy(
               allowGetObjectStatementBuilder.addResource(
                   IamResource.create(getS3UriWithArn(arnPrefix, uri)));
               String bucketArn = arnPrefix + getBucketName(uri);
+              String rawPath = trimLeadingSlash(uri.getPath());
               bucketListStatmentBuilder
                   .computeIfAbsent(
                       bucketArn,
@@ -142,10 +145,14 @@ private IamPolicy createPolicy(
                               .effect(IamEffect.ALLOW)
                               .addAction("s3:ListBucket")
                               .addResource(key))
-                  .addCondition(
+                  .addConditions(
                       IamConditionOperator.STRING_LIKE,
                       "s3:prefix",
-                      concatPathWithSep(trimLeadingSlash(uri.getPath()), "*", "/"));
+                      Arrays.asList(
+                          // Get raw path metadata information for AWS hadoop connector
+                          rawPath,
+                          // Listing objects in raw path
+                          concatPathWithSep(rawPath, "*", "/")));
               bucketGetLocationStatmentBuilder.computeIfAbsent(
                   bucketArn,
                   key ->

bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java

@@ -35,7 +35,8 @@ public class GCSFileSystemProvider implements FileSystemProvider {
 
   @VisibleForTesting
   public static final Map<String, String> GRAVITINO_KEY_TO_GCS_HADOOP_KEY =
-      ImmutableMap.of(GCSProperties.GCS_SERVICE_ACCOUNT_JSON_PATH, GCS_SERVICE_ACCOUNT_JSON_FILE);
+      ImmutableMap.of(
+          GCSProperties.GRAVITINO_GCS_SERVICE_ACCOUNT_FILE, GCS_SERVICE_ACCOUNT_JSON_FILE);
 
   @Override
   public FileSystem getFileSystem(Path path, Map<String, String> config) throws IOException {

catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/GCSProperties.java

@@ -22,7 +22,7 @@
 public class GCSProperties {
 
   // The path of service account JSON file of Google Cloud Storage.
-  public static final String GCS_SERVICE_ACCOUNT_JSON_PATH = "gcs-service-account-file";
+  public static final String GRAVITINO_GCS_SERVICE_ACCOUNT_FILE = "gcs-service-account-file";
 
   private GCSProperties() {}
 }

catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java

@@ -19,7 +19,7 @@
 package org.apache.gravitino.catalog.hadoop.integration.test;
 
 import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS;
-import static org.apache.gravitino.storage.GCSProperties.GCS_SERVICE_ACCOUNT_JSON_PATH;
+import static org.apache.gravitino.storage.GCSProperties.GRAVITINO_GCS_SERVICE_ACCOUNT_FILE;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
@@ -99,7 +99,7 @@ protected String defaultBaseLocation() {
 
   protected void createCatalog() {
     Map<String, String> map = Maps.newHashMap();
-    map.put(GCS_SERVICE_ACCOUNT_JSON_PATH, SERVICE_ACCOUNT_FILE);
+    map.put(GRAVITINO_GCS_SERVICE_ACCOUNT_FILE, SERVICE_ACCOUNT_FILE);
     map.put(FILESYSTEM_PROVIDERS, "gcs");
     metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider, "comment", map);
 
@@ -117,7 +117,7 @@ public void testCreateSchemaAndFilesetWithSpecialLocation() {
     String ossLocation = String.format("gs://%s", BUCKET_NAME);
     Map<String, String> catalogProps = Maps.newHashMap();
     catalogProps.put("location", ossLocation);
-    catalogProps.put(GCS_SERVICE_ACCOUNT_JSON_PATH, SERVICE_ACCOUNT_FILE);
+    catalogProps.put(GRAVITINO_GCS_SERVICE_ACCOUNT_FILE, SERVICE_ACCOUNT_FILE);
     catalogProps.put(FILESYSTEM_PROVIDERS, "gcs");
 
     Catalog localCatalog =

catalogs/catalog-model/build.gradle.kts

@@ -29,32 +29,33 @@ dependencies {
     exclude(group = "*")
   }
 
-  implementation(project(":core")) {
+  implementation(project(":catalogs:catalog-common")) {
     exclude(group = "*")
   }
   implementation(project(":common")) {
     exclude(group = "*")
   }
-
-  implementation(project(":catalogs:catalog-common")) {
+  implementation(project(":core")) {
     exclude(group = "*")
   }
-
   implementation(libs.guava)
   implementation(libs.slf4j.api)
 
   testImplementation(project(":clients:client-java"))
   testImplementation(project(":integration-test-common", "testArtifacts"))
   testImplementation(project(":server"))
   testImplementation(project(":server-common"))
-
   testImplementation(libs.bundles.log4j)
   testImplementation(libs.commons.io)
   testImplementation(libs.commons.lang3)
   testImplementation(libs.mockito.core)
   testImplementation(libs.mockito.inline)
+  testImplementation(libs.mysql.driver)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
+  testImplementation(libs.postgresql.driver)
+  testImplementation(libs.testcontainers)
+  testImplementation(libs.testcontainers.mysql)
 
   testRuntimeOnly(libs.junit.jupiter.engine)
 }
@@ -68,8 +69,9 @@ tasks {
   val copyCatalogLibs by registering(Copy::class) {
     dependsOn("jar", "runtimeJars")
     from("build/libs") {
-      exclude("slf4j-*.jar")
       exclude("guava-*.jar")
+      exclude("log4j-*.jar")
+      exclude("slf4j-*.jar")
     }
     into("$rootDir/distribution/package/catalogs/model/libs")
   }