add flink 1.18 support

apache · Aug 28, 2024 · 38ad59f · 38ad59f
1 parent ae8a6eb
commit 38ad59f
Show file tree

Hide file tree

Showing 12 changed files with 36 additions and 52 deletions.
diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common-format/pom.xml b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common-format/pom.xml
@@ -31,7 +31,7 @@
     <url>https://amoro.apache.org</url>
 
     <properties>
-        <flink.version>1.17.1</flink.version>
+        <flink.version>1.18.1</flink.version>
     </properties>
 
     <dependencies>

diff --git a/...ed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common-iceberg-bridge/pom.xml b/...ed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common-iceberg-bridge/pom.xml
@@ -31,10 +31,9 @@
     <url>https://amoro.apache.org</url>
 
     <properties>
-        <kafka.version>3.2.3</kafka.version>
         <assertj.version>3.21.0</assertj.version>
-        <testcontainers.version>1.17.2</testcontainers.version>
-        <flink.version>1.17.1</flink.version>
+        <testcontainers.version>1.18.3</testcontainers.version>
+        <flink.version>1.18.1</flink.version>
     </properties>
 
     <dependencies>
@@ -58,7 +57,7 @@
 
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-flink-1.17</artifactId>
+            <artifactId>iceberg-flink-1.18</artifactId>
             <version>${iceberg.version}</version>
             <scope>provided</scope>
             <exclusions>
@@ -99,7 +98,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-kafka</artifactId>
-            <version>${flink.version}</version>
+            <version>3.2.0-1.18</version>
             <scope>provided</scope>
         </dependency>
 
@@ -184,7 +183,7 @@
 
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-flink-1.17</artifactId>
+            <artifactId>iceberg-flink-1.18</artifactId>
             <version>${iceberg.version}</version>
             <classifier>tests</classifier>
             <scope>test</scope>

diff --git a/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/pom.xml b/amoro-mixed-format/amoro-mixed-format-flink/amoro-mixed-format-flink-common/pom.xml
@@ -34,10 +34,9 @@
     <packaging>jar</packaging>
 
     <properties>
-        <kafka.version>3.2.3</kafka.version>
         <assertj.version>3.21.0</assertj.version>
-        <testcontainers.version>1.17.2</testcontainers.version>
-        <flink.version>1.17.1</flink.version>
+        <testcontainers.version>1.18.3</testcontainers.version>
+        <flink.version>1.18.1</flink.version>
     </properties>
 
     <dependencies>
@@ -66,7 +65,7 @@
 
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-flink-1.17</artifactId>
+            <artifactId>iceberg-flink-1.18</artifactId>
             <version>${iceberg.version}</version>
             <scope>provided</scope>
             <exclusions>
@@ -119,7 +118,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-kafka</artifactId>
-            <version>${flink.version}</version>
+            <version>3.2.0-1.18</version>
             <scope>provided</scope>
         </dependency>
 
@@ -221,7 +220,7 @@
 
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-flink-1.17</artifactId>
+            <artifactId>iceberg-flink-1.18</artifactId>
             <version>${iceberg.version}</version>
             <classifier>tests</classifier>
             <scope>test</scope>

diff --git a/...ed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/RocksDBCacheState.java b/...ed-format-flink-common/src/main/java/org/apache/amoro/flink/lookup/RocksDBCacheState.java
@@ -23,8 +23,8 @@
 import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.metrics.Gauge;
 import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.shaded.guava30.com.google.common.cache.Cache;
-import org.apache.flink.shaded.guava30.com.google.common.cache.CacheBuilder;
+import org.apache.flink.shaded.guava31.com.google.common.cache.Cache;
+import org.apache.flink.shaded.guava31.com.google.common.cache.CacheBuilder;
 import org.apache.flink.table.data.RowData;
 import org.apache.flink.types.RowKind;
 import org.apache.flink.util.FlinkRuntimeException;

diff --git a/...rmat-flink-common/src/main/java/org/apache/amoro/flink/lookup/RocksDBSetSpilledState.java b/...rmat-flink-common/src/main/java/org/apache/amoro/flink/lookup/RocksDBSetSpilledState.java
@@ -22,7 +22,7 @@
 import org.apache.amoro.utils.map.RocksDBBackend;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.shaded.guava30.com.google.common.cache.Cache;
+import org.apache.flink.shaded.guava31.com.google.common.cache.Cache;
 import org.apache.flink.table.data.RowData;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

diff --git a/...-format-flink-common/src/main/java/org/apache/amoro/flink/read/internals/KafkaSource.java b/...-format-flink-common/src/main/java/org/apache/amoro/flink/read/internals/KafkaSource.java
@@ -179,7 +179,7 @@ public SplitEnumerator<KafkaPartitionSplit, KafkaSourceEnumState> restoreEnumera
         props,
         enumContext,
         boundedness,
-        checkpoint.assignedPartitions());
+        checkpoint);
   }
 
   @Internal

diff --git a/...ro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/lookup/TestKVTable.java b/...ro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/lookup/TestKVTable.java
@@ -30,9 +30,9 @@
 import org.apache.flink.core.memory.DataInputDeserializer;
 import org.apache.flink.core.memory.DataOutputSerializer;
 import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
-import org.apache.flink.shaded.guava30.com.google.common.cache.Cache;
-import org.apache.flink.shaded.guava30.com.google.common.cache.CacheBuilder;
-import org.apache.flink.shaded.guava30.com.google.common.collect.Lists;
+import org.apache.flink.shaded.guava31.com.google.common.cache.Cache;
+import org.apache.flink.shaded.guava31.com.google.common.cache.CacheBuilder;
+import org.apache.flink.shaded.guava31.com.google.common.collect.Lists;
 import org.apache.flink.table.catalog.Column;
 import org.apache.flink.table.catalog.ResolvedSchema;
 import org.apache.flink.table.catalog.UniqueConstraint;

diff --git a/...on/src/test/java/org/apache/amoro/flink/read/hybrid/reader/TestRowDataReaderFunction.java b/...on/src/test/java/org/apache/amoro/flink/read/hybrid/reader/TestRowDataReaderFunction.java
@@ -31,7 +31,7 @@
 import org.apache.amoro.scan.MixedFileScanTask;
 import org.apache.amoro.table.KeyedTable;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.shaded.guava30.com.google.common.collect.Maps;
+import org.apache.flink.shaded.guava31.com.google.common.collect.Maps;
 import org.apache.flink.table.data.GenericRowData;
 import org.apache.flink.table.data.RowData;
 import org.apache.flink.table.data.StringData;

diff --git a/.../amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestJoin.java b/.../amoro-mixed-format-flink-common/src/test/java/org/apache/amoro/flink/table/TestJoin.java
@@ -32,7 +32,7 @@
 import org.apache.amoro.table.TableIdentifier;
 import org.apache.flink.api.common.JobStatus;
 import org.apache.flink.runtime.testutils.CommonTestUtils;
-import org.apache.flink.shaded.guava30.com.google.common.collect.Lists;
+import org.apache.flink.shaded.guava31.com.google.common.collect.Lists;
 import org.apache.flink.table.api.DataTypes;
 import org.apache.flink.table.api.TableResult;
 import org.apache.flink.table.api.TableSchema;

diff --git a/amoro-mixed-format/amoro-mixed-format-flink/pom.xml b/amoro-mixed-format/amoro-mixed-format-flink/pom.xml
@@ -37,15 +37,14 @@
         <module>amoro-mixed-format-flink-common</module>
         <module>amoro-mixed-format-flink-common-format</module>
         <module>amoro-mixed-format-flink-common-iceberg-bridge</module>
-        <module>v1.17/amoro-mixed-format-flink-1.17</module>
-        <module>v1.17/amoro-mixed-format-flink-runtime-1.17</module>
+        <module>v1.18/amoro-mixed-format-flink-1.18</module>
+        <module>v1.18/amoro-mixed-format-flink-runtime-1.18</module>
     </modules>
 
     <properties>
         <!--    default flink version    -->
-        <flink.version>1.17.2</flink.version>
-        <kafka.version>2.4.1</kafka.version>
-        <gson.version>2.9.0</gson.version>
-        <jackson.vesion>2.10.2</jackson.vesion>
+        <flink.version>1.18.1</flink.version>
+        <gson.version>2.11.0</gson.version>
+        <jackson.vesion>2.15.3</jackson.vesion>
     </properties>
 </project>
diff --git a/....17/amoro-mixed-format-flink-1.17/pom.xml → ....18/amoro-mixed-format-flink-1.18/pom.xml b/....17/amoro-mixed-format-flink-1.17/pom.xml → ....18/amoro-mixed-format-flink-1.18/pom.xml
@@ -27,17 +27,16 @@
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
-    <artifactId>amoro-mixed-format-flink-1.17</artifactId>
-    <name>Amoro Project Mixed Format Flink 1.17</name>
+    <artifactId>amoro-mixed-format-flink-1.18</artifactId>
+    <name>Amoro Project Mixed Format Flink 1.18</name>
     <url>https://amoro.apache.org</url>
 
     <packaging>jar</packaging>
 
     <properties>
-        <kafka.version>3.2.3</kafka.version>
         <assertj.version>3.21.0</assertj.version>
-        <testcontainers.version>1.17.2</testcontainers.version>
-        <flink.version>1.17.2</flink.version>
+        <testcontainers.version>1.18.3</testcontainers.version>
+        <flink.version>1.18.1</flink.version>
     </properties>
 
     <dependencies>
@@ -49,7 +48,7 @@
 
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-flink-1.17</artifactId>
+            <artifactId>iceberg-flink-1.18</artifactId>
             <version>${iceberg.version}</version>
             <exclusions>
                 <exclusion>
@@ -69,7 +68,7 @@
 
         <dependency>
             <groupId>org.apache.paimon</groupId>
-            <artifactId>paimon-flink-1.17</artifactId>
+            <artifactId>paimon-flink-1.18</artifactId>
             <version>${paimon.version}</version>
         </dependency>
     </dependencies>

diff --git a/...o-mixed-format-flink-runtime-1.17/pom.xml → ...o-mixed-format-flink-runtime-1.18/pom.xml b/...o-mixed-format-flink-runtime-1.17/pom.xml → ...o-mixed-format-flink-runtime-1.18/pom.xml
@@ -27,37 +27,25 @@
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
-    <artifactId>amoro-mixed-format-flink-runtime-1.17</artifactId>
-    <name>Amoro Project Mixed Format Flink 1.17 Runtime</name>
+    <artifactId>amoro-mixed-format-flink-runtime-1.18</artifactId>
+    <name>Amoro Project Mixed Format Flink 1.18 Runtime</name>
     <url>https://amoro.apache.org</url>
 
     <properties>
-        <flink.version>1.17.2</flink.version>
+        <flink.version>1.18.1</flink.version>
     </properties>
 
     <dependencies>
         <dependency>
             <groupId>org.apache.amoro</groupId>
-            <artifactId>amoro-mixed-format-flink-1.17</artifactId>
+            <artifactId>amoro-mixed-format-flink-1.18</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
 
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-connector-kafka</artifactId>
-            <version>${flink.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.github.luben</groupId>
-                    <artifactId>zstd-jni</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka-clients</artifactId>
-            <version>${kafka.version}</version>
+            <version>3.2.0-1.18</version>
             <exclusions>
                 <exclusion>
                     <groupId>com.github.luben</groupId>