Skip to content

Commit 1b9c77b

Browse files
yuxiqianyuxiqian
committed
mig-test: fix datastream migration failure
1 parent 6934d01 commit 1b9c77b

File tree

11 files changed

+356
-62
lines changed

11 files changed

+356
-62
lines changed

‎tools/mig-test/datastream/compile_jobs.rb

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
JOB_VERSIONS.each do |version|
2222
puts "Compiling DataStream job for CDC #{version}"
23-
`cd datastream-#{version} && mvn clean package -DskipTests`
23+
system "cd datastream-#{version} && mvn clean package -DskipTests"
2424
end
2525

2626
puts 'Done'

‎tools/mig-test/datastream/datastream-3.2.0/pom.xml

+84-8
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ limitations under the License.
2929
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
3030
<flink.version>1.18.1</flink.version>
3131
<flink.cdc.version>3.2.0</flink.cdc.version>
32-
<debezium.version>1.9.7.Final</debezium.version>
32+
<debezium.version>1.9.8.Final</debezium.version>
3333
<scala.binary.version>2.12</scala.binary.version>
3434
<slf4j.version>2.0.13</slf4j.version>
3535
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -136,20 +136,96 @@ limitations under the License.
136136
</configuration>
137137
</plugin>
138138
<plugin>
139-
<artifactId>maven-assembly-plugin</artifactId>
139+
<groupId>org.apache.maven.plugins</groupId>
140+
<artifactId>maven-shade-plugin</artifactId>
140141
<executions>
141142
<execution>
143+
<id>shade-flink</id>
142144
<phase>package</phase>
143145
<goals>
144-
<goal>single</goal>
146+
<goal>shade</goal>
145147
</goals>
148+
<configuration>
149+
<!-- Shading test jar have bug in some previous version, so close this configuration here,
150+
see https://issues.apache.org/jira/browse/MSHADE-284 -->
151+
<shadeTestJar>false</shadeTestJar>
152+
<shadedArtifactAttached>false</shadedArtifactAttached>
153+
<createDependencyReducedPom>true</createDependencyReducedPom>
154+
<dependencyReducedPomLocation>
155+
${project.basedir}/target/dependency-reduced-pom.xml
156+
</dependencyReducedPomLocation>
157+
<filters combine.children="append">
158+
<filter>
159+
<artifact>*:*</artifact>
160+
<excludes>
161+
<exclude>module-info.class</exclude>
162+
<exclude>META-INF/*.SF</exclude>
163+
<exclude>META-INF/*.DSA</exclude>
164+
<exclude>META-INF/*.RSA</exclude>
165+
</excludes>
166+
</filter>
167+
</filters>
168+
<artifactSet>
169+
<includes>
170+
<!-- include nothing -->
171+
<include>io.debezium:debezium-api</include>
172+
<include>io.debezium:debezium-embedded</include>
173+
<include>io.debezium:debezium-core</include>
174+
<include>io.debezium:debezium-ddl-parser</include>
175+
<include>io.debezium:debezium-connector-mysql</include>
176+
<include>org.apache.flink:flink-connector-debezium</include>
177+
<include>org.apache.flink:flink-connector-mysql-cdc</include>
178+
<include>org.antlr:antlr4-runtime</include>
179+
<include>org.apache.kafka:*</include>
180+
<include>mysql:mysql-connector-java</include>
181+
<include>com.zendesk:mysql-binlog-connector-java</include>
182+
<include>com.fasterxml.*:*</include>
183+
<include>com.google.guava:*</include>
184+
<include>com.esri.geometry:esri-geometry-api</include>
185+
<include>com.zaxxer:HikariCP</include>
186+
<!-- Include fixed version 30.1.1-jre-16.0 of flink shaded guava -->
187+
<include>org.apache.flink:flink-shaded-guava</include>
188+
</includes>
189+
</artifactSet>
190+
<relocations>
191+
<relocation>
192+
<pattern>org.apache.kafka</pattern>
193+
<shadedPattern>
194+
org.apache.flink.cdc.connectors.shaded.org.apache.kafka
195+
</shadedPattern>
196+
</relocation>
197+
<relocation>
198+
<pattern>org.antlr</pattern>
199+
<shadedPattern>
200+
org.apache.flink.cdc.connectors.shaded.org.antlr
201+
</shadedPattern>
202+
</relocation>
203+
<relocation>
204+
<pattern>com.fasterxml</pattern>
205+
<shadedPattern>
206+
org.apache.flink.cdc.connectors.shaded.com.fasterxml
207+
</shadedPattern>
208+
</relocation>
209+
<relocation>
210+
<pattern>com.google</pattern>
211+
<shadedPattern>
212+
org.apache.flink.cdc.connectors.shaded.com.google
213+
</shadedPattern>
214+
</relocation>
215+
<relocation>
216+
<pattern>com.esri.geometry</pattern>
217+
<shadedPattern>org.apache.flink.cdc.connectors.shaded.com.esri.geometry</shadedPattern>
218+
</relocation>
219+
<relocation>
220+
<pattern>com.zaxxer</pattern>
221+
<shadedPattern>
222+
org.apache.flink.cdc.connectors.shaded.com.zaxxer
223+
</shadedPattern>
224+
</relocation>
225+
</relocations>
226+
</configuration>
146227
</execution>
147228
</executions>
148-
<configuration>
149-
<descriptorRefs>
150-
<descriptorRef>jar-with-dependencies</descriptorRef>
151-
</descriptorRefs>
152-
</configuration>
153229
</plugin>
154230
</plugins>
155231
</build>

‎tools/mig-test/datastream/datastream-3.2.0/src/main/java/DataStreamJob.java

+2-6
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323

2424
public class DataStreamJob {
2525

26-
public static void main(String[] args) {
26+
public static void main(String[] args) throws Exception {
2727
MySqlSource<String> mySqlSource = MySqlSource.<String>builder()
2828
.hostname("localhost")
2929
.port(3306)
@@ -45,10 +45,6 @@ public static void main(String[] args) {
4545
.print()
4646
.setParallelism(1);
4747

48-
try {
49-
env.execute();
50-
} catch (Exception e) {
51-
// ... unfortunately
52-
}
48+
env.execute();
5349
}
5450
}

‎tools/mig-test/datastream/datastream-3.2.1/pom.xml

+83-7
Original file line numberDiff line numberDiff line change
@@ -136,20 +136,96 @@ limitations under the License.
136136
</configuration>
137137
</plugin>
138138
<plugin>
139-
<artifactId>maven-assembly-plugin</artifactId>
139+
<groupId>org.apache.maven.plugins</groupId>
140+
<artifactId>maven-shade-plugin</artifactId>
140141
<executions>
141142
<execution>
143+
<id>shade-flink</id>
142144
<phase>package</phase>
143145
<goals>
144-
<goal>single</goal>
146+
<goal>shade</goal>
145147
</goals>
148+
<configuration>
149+
<!-- Shading test jar have bug in some previous version, so close this configuration here,
150+
see https://issues.apache.org/jira/browse/MSHADE-284 -->
151+
<shadeTestJar>false</shadeTestJar>
152+
<shadedArtifactAttached>false</shadedArtifactAttached>
153+
<createDependencyReducedPom>true</createDependencyReducedPom>
154+
<dependencyReducedPomLocation>
155+
${project.basedir}/target/dependency-reduced-pom.xml
156+
</dependencyReducedPomLocation>
157+
<filters combine.children="append">
158+
<filter>
159+
<artifact>*:*</artifact>
160+
<excludes>
161+
<exclude>module-info.class</exclude>
162+
<exclude>META-INF/*.SF</exclude>
163+
<exclude>META-INF/*.DSA</exclude>
164+
<exclude>META-INF/*.RSA</exclude>
165+
</excludes>
166+
</filter>
167+
</filters>
168+
<artifactSet>
169+
<includes>
170+
<!-- include nothing -->
171+
<include>io.debezium:debezium-api</include>
172+
<include>io.debezium:debezium-embedded</include>
173+
<include>io.debezium:debezium-core</include>
174+
<include>io.debezium:debezium-ddl-parser</include>
175+
<include>io.debezium:debezium-connector-mysql</include>
176+
<include>org.apache.flink:flink-connector-debezium</include>
177+
<include>org.apache.flink:flink-connector-mysql-cdc</include>
178+
<include>org.antlr:antlr4-runtime</include>
179+
<include>org.apache.kafka:*</include>
180+
<include>mysql:mysql-connector-java</include>
181+
<include>com.zendesk:mysql-binlog-connector-java</include>
182+
<include>com.fasterxml.*:*</include>
183+
<include>com.google.guava:*</include>
184+
<include>com.esri.geometry:esri-geometry-api</include>
185+
<include>com.zaxxer:HikariCP</include>
186+
<!-- Include fixed version 30.1.1-jre-16.0 of flink shaded guava -->
187+
<include>org.apache.flink:flink-shaded-guava</include>
188+
</includes>
189+
</artifactSet>
190+
<relocations>
191+
<relocation>
192+
<pattern>org.apache.kafka</pattern>
193+
<shadedPattern>
194+
org.apache.flink.cdc.connectors.shaded.org.apache.kafka
195+
</shadedPattern>
196+
</relocation>
197+
<relocation>
198+
<pattern>org.antlr</pattern>
199+
<shadedPattern>
200+
org.apache.flink.cdc.connectors.shaded.org.antlr
201+
</shadedPattern>
202+
</relocation>
203+
<relocation>
204+
<pattern>com.fasterxml</pattern>
205+
<shadedPattern>
206+
org.apache.flink.cdc.connectors.shaded.com.fasterxml
207+
</shadedPattern>
208+
</relocation>
209+
<relocation>
210+
<pattern>com.google</pattern>
211+
<shadedPattern>
212+
org.apache.flink.cdc.connectors.shaded.com.google
213+
</shadedPattern>
214+
</relocation>
215+
<relocation>
216+
<pattern>com.esri.geometry</pattern>
217+
<shadedPattern>org.apache.flink.cdc.connectors.shaded.com.esri.geometry</shadedPattern>
218+
</relocation>
219+
<relocation>
220+
<pattern>com.zaxxer</pattern>
221+
<shadedPattern>
222+
org.apache.flink.cdc.connectors.shaded.com.zaxxer
223+
</shadedPattern>
224+
</relocation>
225+
</relocations>
226+
</configuration>
146227
</execution>
147228
</executions>
148-
<configuration>
149-
<descriptorRefs>
150-
<descriptorRef>jar-with-dependencies</descriptorRef>
151-
</descriptorRefs>
152-
</configuration>
153229
</plugin>
154230
</plugins>
155231
</build>

‎tools/mig-test/datastream/datastream-3.2.1/src/main/java/DataStreamJob.java

+2-6
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323

2424
public class DataStreamJob {
2525

26-
public static void main(String[] args) {
26+
public static void main(String[] args) throws Exception {
2727
MySqlSource<String> mySqlSource = MySqlSource.<String>builder()
2828
.hostname("localhost")
2929
.port(3306)
@@ -45,10 +45,6 @@ public static void main(String[] args) {
4545
.print()
4646
.setParallelism(1);
4747

48-
try {
49-
env.execute();
50-
} catch (Exception e) {
51-
// ... unfortunately
52-
}
48+
env.execute();
5349
}
5450
}

0 commit comments

Comments
 (0)