HIVE-28059 : major rebase stage 1;

apache · Jan 31, 2025 · fb43336 · fb43336
1 parent d11bc4b
commit fb43336
Show file tree

Hide file tree

Showing 14 changed files with 207 additions and 57 deletions.
diff --git a/iceberg/patched-iceberg-api/pom.xml b/iceberg/patched-iceberg-api/pom.xml
@@ -21,7 +21,6 @@
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>patched-iceberg-api</artifactId>
-  <version>${revision}</version>
   <name>Patched Iceberg API</name>
   <!-- Temporary package until we need unreleased Iceberg changes for development purposes. -->
   <!-- We should periodically clean this package during rebasing to newer Iceberg releases. -->

diff --git a/iceberg/patched-iceberg-core/pom.xml b/iceberg/patched-iceberg-core/pom.xml
@@ -22,7 +22,6 @@
   <modelVersion>4.0.0</modelVersion>
   <artifactId>patched-iceberg-core</artifactId>
   <name>Patched Iceberg Core</name>
-  <version>${revision}</version>
   <!-- Temporary package until we need unreleased Iceberg changes for development purposes. -->
   <!-- We should periodically clean this package during rebasing to newer Iceberg releases. -->
   <!-- We should remove the package for good when Iceberg integration has been stabilized. -->

diff --git a/pom.xml b/pom.xml
@@ -99,7 +99,7 @@
     <maven.exec.plugin.version>3.1.0</maven.exec.plugin.version>
     <maven.versions.plugin.version>2.16.0</maven.versions.plugin.version>
     <maven.shade.plugin.version>3.5.0</maven.shade.plugin.version>
-    <maven.surefire.plugin.version>3.0.0-M4</maven.surefire.plugin.version>
+    <maven.surefire.plugin.version>3.5.1</maven.surefire.plugin.version>
     <maven.cyclonedx.plugin.version>2.7.10</maven.cyclonedx.plugin.version>
     <maven.license.plugin.version>2.3.0</maven.license.plugin.version>
     <!-- Library Dependency Versions -->
@@ -110,7 +110,7 @@
     <antlr4.version>4.9.3</antlr4.version>
     <apache-directory-server.version>1.5.7</apache-directory-server.version>
     <!-- Include arrow for LlapOutputFormatService -->
-    <arrow.version>12.0.0</arrow.version>
+    <arrow.version>16.0.0</arrow.version>
     <avatica.version>1.12.0</avatica.version>
     <avro.version>1.11.4</avro.version>
     <bcprov-jdk18on.version>1.78</bcprov-jdk18on.version>
@@ -126,7 +126,7 @@
     <commons-compress.version>1.26.0</commons-compress.version>
     <commons-configuration.version>1.10</commons-configuration.version>
     <commons-exec.version>1.1</commons-exec.version>
-    <commons-io.version>2.12.0</commons-io.version>
+    <commons-io.version>2.14.0</commons-io.version>
     <commons-lang3.version>3.12.0</commons-lang3.version>
     <commons-math3.version>3.6.1</commons-math3.version>
     <commons-dbcp2.version>2.12.0</commons-dbcp2.version>
@@ -151,7 +151,6 @@
     <!-- httpcomponents are not always in version sync -->
     <httpcomponents.client.version>4.5.13</httpcomponents.client.version>
     <httpcomponents.core.version>4.4.13</httpcomponents.core.version>
-    <iceberg.version>1.6.1</iceberg.version>
     <ivy.version>2.5.2</ivy.version>
     <jackson.version>2.16.1</jackson.version>
     <jamon.plugin.version>2.3.4</jamon.plugin.version>
@@ -167,19 +166,19 @@
     <jodd.version>6.0.0</jodd.version>
     <json.version>1.8</json.version>
     <junit.version>4.13.2</junit.version>
-    <junit.jupiter.version>5.10.0</junit.jupiter.version>
-    <junit.vintage.version>5.6.3</junit.vintage.version>
+    <junit.jupiter.version>5.11.2</junit.jupiter.version>
+    <junit.vintage.version>5.11.2</junit.vintage.version>
     <kafka.version>2.5.0</kafka.version>
     <kryo.version>5.5.0</kryo.version>
     <reflectasm.version>1.11.9</reflectasm.version>
     <kudu.version>1.12.0</kudu.version>
     <!-- Leaving libfb303 at 0.9.3 regardless of libthrift: As per THRIFT-4613 The Apache Thrift project does not publish items related to fb303 at this point -->
     <libfb303.version>0.9.3</libfb303.version>
     <libthrift.version>0.16.0</libthrift.version>
-    <log4j2.version>2.24.1</log4j2.version>
+    <log4j2.version>2.18.0</log4j2.version>
     <mariadb.version>2.5.0</mariadb.version>
     <mssql.version>6.2.1.jre8</mssql.version>
-    <mysql.version>8.0.31</mysql.version>
+    <mysql.version>8.2.0</mysql.version>
     <postgres.version>42.7.3</postgres.version>
     <oracle.version>21.3.0.0</oracle.version>
     <opencsv.version>5.9</opencsv.version>
@@ -188,12 +187,12 @@
     <mockito-core.version>3.4.4</mockito-core.version>
     <mockito-inline.version>4.11.0</mockito-inline.version>
     <mina.version>2.0.0-M5</mina.version>
-    <netty.version>4.1.77.Final</netty.version>
+    <netty.version>4.1.116.Final</netty.version>
     <netty3.version>3.10.5.Final</netty3.version>
     <!-- used by druid storage handler -->
     <pac4j-saml.version>4.5.5</pac4j-saml.version>
     <paranamer.version>2.8</paranamer.version>
-    <parquet.version>1.13.1</parquet.version>
+    <parquet.version>1.14.4</parquet.version>
     <pig.version>0.16.0</pig.version>
     <plexus.version>1.5.6</plexus.version>
     <protobuf.version>3.25.5</protobuf.version>
@@ -1016,7 +1015,7 @@
         <exclusions>
           <exclusion>
             <groupId>io.netty</groupId>
-            <artifactId>netty</artifactId>
+            <artifactId>netty-all</artifactId>
           </exclusion>
         </exclusions>
       </dependency>
@@ -1754,6 +1753,7 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
+          <enableOutErrElements>false</enableOutErrElements>
           <excludes>
             <exclude>**/TestSerDe.java</exclude>
             <exclude>**/TestHiveMetaStore.java</exclude>
@@ -1765,8 +1765,8 @@
           </excludes>
           <redirectTestOutputToFile>true</redirectTestOutputToFile>
           <reuseForks>false</reuseForks>
-          <failIfNoTests>false</failIfNoTests>
-          <argLine>${maven.test.jvm.args}</argLine>
+          <failIfNoSpecifiedTests>false</failIfNoSpecifiedTests>
+          <argLine>${maven.test.jvm.args} -Xshare:off</argLine>
           <trimStackTrace>false</trimStackTrace>
           <additionalClasspathElements>
             <additionalClasspathElement>${test.conf.dir}</additionalClasspathElement>
@@ -1795,10 +1795,10 @@
             <!-- required for hive-exec jar path and tests which reference a jar -->
             <maven.local.repository>${maven.repo.local}</maven.local.repository>
             <mapred.job.tracker>local</mapred.job.tracker>
-            <log4j.configurationFile>${test.log4j.scheme}${test.conf.dir}/hive-log4j2.properties
-            </log4j.configurationFile>
+            <log4j.configurationFile>${test.log4j.scheme}${test.conf.dir}/hive-log4j2.properties</log4j.configurationFile>
             <hive.test.console.log.level>${test.console.log.level}</hive.test.console.log.level>
             <hive.cluster.id>hive-test-cluster-id-cli</hive.cluster.id>
+            <log4j.debug>true</log4j.debug>
             <!-- don't dirty up /tmp -->
             <java.io.tmpdir>${test.tmp.dir}</java.io.tmpdir>
             <!-- Hadoop's minidfs class uses this -->
@@ -1916,11 +1916,9 @@
                   <workingDirectory>${basedir}</workingDirectory>
                   <arguments>
                     <argument>-c</argument>
-                    <argument>${thrift.home}/bin/thrift -version | fgrep 'Thrift version ${libthrift.version}' &amp;&amp;
-                      exit 0;
+                    <argument>${thrift.home}/bin/thrift -version | fgrep 'Thrift version ${libthrift.version}' &amp;&amp; exit 0;
                       echo "=================================================================================";
-                      echo "========== [FATAL] Build is configured to require Thrift version ${libthrift.version}
-                      =========";
+                      echo "========== [FATAL] Build is configured to require Thrift version ${libthrift.version} =========";
                       echo "========== Currently installed: ";
                       ${thrift.home}/bin/thrift -version;
                       echo "=================================================================================";

diff --git a/...e-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/...e-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
@@ -50,14 +50,9 @@
 
 public class HMSCatalogServer {
   private static final String CACHE_EXPIRY = "hive.metastore.catalog.cache.expiry";
-  private static final String CACHE_AUTHORIZATION = "hive.metastore.catalog.cache.authorization";
   private static final String JETTY_THREADPOOL_MIN = "hive.metastore.catalog.jetty.threadpool.min";
   private static final String JETTY_THREADPOOL_MAX = "hive.metastore.catalog.jetty.threadpool.max";
   private static final String JETTY_THREADPOOL_IDLE = "hive.metastore.catalog.jetty.threadpool.idle";
-  /**
-   * The metric names prefix.
-   */
-  static final String HMS_METRIC_PREFIX = "hmscatalog.";
   private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogServer.class);
   private static Reference<Catalog> catalogRef;
 
@@ -92,12 +87,12 @@ public static Catalog createCatalog(Configuration configuration) {
       properties.put("external-warehouse", cextwarehouse);
     }
     catalog.initialize("hive", properties);
-    long expiry = 0;//configuration.getLong(CACHE_EXPIRY, 60_000L);
+    long expiry = configuration.getLong(CACHE_EXPIRY, 60_000L);
     return expiry > 0? HiveCachingCatalog.wrap(catalog, expiry) : catalog;
   }
 
   public static HttpServlet createServlet(Configuration configuration, Catalog catalog) throws IOException {
-    String auth = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.CATALOG_SERVLET_AUTH);
+    String auth = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_AUTH);
     boolean jwt = "jwt".equalsIgnoreCase(auth);
     SecureServletCaller security = new ServletSecurity(configuration, jwt);
     Catalog actualCatalog = catalog;
@@ -112,11 +107,12 @@ public static HttpServlet createServlet(Configuration configuration, Catalog cat
   /**
    * Convenience method to start a http server that only serves this servlet.
    * @param conf the configuration
+   * @param catalog the catalog instance to serve
    * @return the server instance
    * @throws Exception if servlet initialization fails
    */
   public static Server startServer(Configuration conf, HiveCatalog catalog) throws Exception {
-    int port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.CATALOG_SERVLET_PORT);
+    int port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT);
     if (port < 0) {
       return null;
     }
@@ -125,7 +121,7 @@ public static Server startServer(Configuration conf, HiveCatalog catalog) throws
     context.setContextPath("/");
     ServletHolder servletHolder = new ServletHolder(servlet);
     servletHolder.setInitParameter("javax.ws.rs.Application", "ServiceListPublic");
-    final String cli = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CATALOG_SERVLET_PATH);
+    final String cli = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PATH);
     context.addServlet(servletHolder, "/" + cli + "/*");
     context.setVirtualHosts(null);
     context.setGzipHandler(new GzipHandler());

diff --git a/...dalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/...dalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
@@ -159,9 +159,9 @@ public void setUp() throws Exception {
     MetastoreConf.setVar(conf, MetastoreConf.ConfVars.SCHEMA_INFO_CLASS, "org.apache.iceberg.rest.HMSTestBase$TestSchemaInfo");
     // Events that get cleaned happen in batches of 1 to exercise batching code
     MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.EVENT_CLEAN_MAX_EVENTS, 1L);
-    MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.CATALOG_SERVLET_PORT, 0);
-    MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_SERVLET_AUTH, "jwt");
-    MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_SERVLET_PATH, catalogPath);
+    MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT, 0);
+    MetastoreConf.setVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_AUTH, "jwt");
+    MetastoreConf.setVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PATH, catalogPath);
     MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_METASTORE_AUTHENTICATION_JWT_JWKS_URL,
         "http://localhost:" + MOCK_JWKS_SERVER_PORT + "/jwks");
     MOCK_JWKS_SERVER.stubFor(get("/jwks")

diff --git a/...one-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java b/...one-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
@@ -31,10 +31,6 @@
 import org.apache.iceberg.Transaction;
 import org.apache.iceberg.catalog.Namespace;
 import org.apache.iceberg.catalog.TableIdentifier;
-import static org.apache.iceberg.rest.HMSTestBase.DB_NAME;
-import static org.apache.iceberg.rest.HMSTestBase.RND;
-import static org.apache.iceberg.rest.HMSTestBase.clientCall;
-import static org.apache.iceberg.rest.HMSTestBase.reportMetricCounters;
 import org.apache.iceberg.types.Types;
 import static org.apache.iceberg.types.Types.NestedField.required;
 import org.junit.Assert;
@@ -59,11 +55,6 @@ public void tearDown() throws Exception {
       super.tearDown();
   }
 
-  @Test
-  public void testC0() throws Exception {
-      testCreateNamespaceHttp();
-  }
-
   @Test
   public void testCreateNamespaceHttp() throws Exception {
     String ns = "nstesthttp";
@@ -138,7 +129,6 @@ public void testCreateTableTxnBuilder() throws Exception {
       Object response = clientCall(jwt, url, "GET", null);
       Assert.assertNotNull(response);
 
-if (false) {
       // list tables in hivedb
       url = new URL("http://hive@localhost:" + catalogPort + "/" + catalogPath+"/v1/namespaces/" + DB_NAME + "/tables");
       // succeed
@@ -155,7 +145,6 @@ public void testCreateTableTxnBuilder() throws Exception {
       // quick check on metrics
       Map<String, Long> counters = reportMetricCounters("list_namespaces", "list_tables", "load_table");
       counters.forEach((key, value) -> Assert.assertTrue(key, value > 0));
-}
       table = catalog.loadTable(tableIdent);
       Assert.assertNotNull(table);
     } catch (Exception xany) {

diff --git a/...e/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/...e/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1827,6 +1827,20 @@ public enum ConfVars {
         "hive.metastore.properties.servlet.auth", "jwt",
         "Property-maps servlet authentication method (simple or jwt)."
     ),
+    ICEBERG_CATALOG_SERVLET_PATH("hive.metastore.catalog.servlet.path",
+        "hive.metastore.catalog.servlet.path", "icecli",
+        "HMS Iceberg Catalog servlet path component of URL endpoint."
+    ),
+    ICEBERG_CATALOG_SERVLET_PORT("hive.metastore.catalog.servlet.port",
+        "hive.metastore.catalog.servlet.port", -1,
+        "HMS Iceberg Catalog servlet server port. Negative value disables the servlet," +
+            " 0 will let the system determine the catalog server port," +
+            " positive value will be used as-is."
+    ),
+    ICEBERG_CATALOG_SERVLET_AUTH("hive.metastore.catalog.servlet.auth",
+        "hive.metastore.catalog.servlet.auth", "jwt",
+        "HMS Iceberg Catalog servlet authentication method (simple or jwt)."
+    ),
 
     // Deprecated Hive values that we are keeping for backwards compatibility.
     @Deprecated

diff --git a/...-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/SummaryMapBuilder.java b/...-common/src/main/java/org/apache/hadoop/hive/metastore/metasummary/SummaryMapBuilder.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.metasummary;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class SummaryMapBuilder {
+  private final Map<String, Object> container = new HashMap<>();
+
+  public SummaryMapBuilder add(String key, Object value) {
+    container.put(key, value);
+    return this;
+  }
+
+  public Map<String, Object> build() {
+    Map<String, Object> result = new HashMap<>(container);
+    container.clear();
+    return result;
+  }
+
+  public <T> T get(String key, Class<T> type) {
+    if (!container.containsKey(key)) {
+      return null;
+    }
+    return type.cast(container.get(key));
+  }
+}
diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml
@@ -12,8 +12,7 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <parent>
     <artifactId>hive-standalone-metastore</artifactId>
     <groupId>org.apache.hive</groupId>