Skip to content

Commit 3d77dfc

Browse files
yuxiqianyuxiqian
committed
fix: mysql ci failure
1 parent 9e98c49 commit 3d77dfc

File tree

5 files changed

+77
-75
lines changed

5 files changed

+77
-75
lines changed

‎flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlSourceITCase.java

+12-17
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@
9696
import java.util.concurrent.FutureTask;
9797
import java.util.concurrent.TimeUnit;
9898
import java.util.concurrent.TimeoutException;
99-
import java.util.stream.Collectors;
10099

101100
import static java.lang.String.format;
102101
import static org.apache.flink.api.common.JobStatus.RUNNING;
@@ -347,7 +346,10 @@ public void testSnapshotSplitReadingFailCrossCheckpoints() throws Exception {
347346
// Check all snapshot records are sent with exactly-once semantics
348347
assertEqualsInAnyOrder(
349348
Arrays.asList(expectedSnapshotData),
350-
fetchAndConvert(iterator, expectedSnapshotData.length, RowData::toString));
349+
fetchAndConvert(
350+
iterator,
351+
expectedSnapshotData.length,
352+
MySqlSourceITCase::convertRowDataToRowString));
351353
assertTrue(!hasNextData(iterator));
352354
jobClient.cancel().get();
353355
}
@@ -1073,26 +1075,19 @@ private void checkBinlogData(
10731075
assertThat(iterator.hasNext()).isFalse();
10741076
}
10751077

1076-
private static List<String> convertRowDataToRowString(List<RowData> rows) {
1078+
private static String convertRowDataToRowString(RowData row) {
10771079
LinkedHashMap<String, Integer> map = new LinkedHashMap<>();
10781080
map.put("id", 0);
10791081
map.put("name", 1);
10801082
map.put("address", 2);
10811083
map.put("phone_number", 3);
1082-
return rows.stream()
1083-
.map(
1084-
row ->
1085-
RowUtils.createRowWithNamedPositions(
1086-
row.getRowKind(),
1087-
new Object[] {
1088-
row.getLong(0),
1089-
row.getString(1),
1090-
row.getString(2),
1091-
row.getString(3)
1092-
},
1093-
map)
1094-
.toString())
1095-
.collect(Collectors.toList());
1084+
return RowUtils.createRowWithNamedPositions(
1085+
row.getRowKind(),
1086+
new Object[] {
1087+
row.getLong(0), row.getString(1), row.getString(2), row.getString(3)
1088+
},
1089+
map)
1090+
.toString();
10961091
}
10971092

10981093
private String getTableNameRegex(String[] captureCustomerTables) {

‎flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/NewlyAddedTableITCase.java

+21-18
Original file line numberDiff line numberDiff line change
@@ -488,7 +488,10 @@ public void testRemoveAndAddNewTable() throws Exception {
488488
.collect(Collectors.toList())
489489
: expectedCustomersResult;
490490
List<String> rows =
491-
fetchAndConvert(iterator, expectedSnapshotResult.size(), RowData::toString);
491+
fetchAndConvert(
492+
iterator,
493+
expectedSnapshotResult.size(),
494+
NewlyAddedTableITCase::convertRowDataToRowString);
492495
assertEqualsInAnyOrder(expectedSnapshotResult, rows);
493496

494497
// make binlog events
@@ -504,7 +507,11 @@ public void testRemoveAndAddNewTable() throws Exception {
504507
"UPDATE " + tableId + " SET address = 'Update2' where id = 103");
505508
connection.commit();
506509
}
507-
rows = fetchAndConvert(iterator, expectedBinlogResult.size(), RowData::toString);
510+
rows =
511+
fetchAndConvert(
512+
iterator,
513+
expectedBinlogResult.size(),
514+
NewlyAddedTableITCase::convertRowDataToRowString);
508515
assertEqualsInAnyOrder(expectedBinlogResult, rows);
509516

510517
finishedSavePointPath = triggerSavepointWithRetry(jobClient, savepointDirectory);
@@ -541,28 +548,24 @@ protected CollectResultIterator<RowData> addCollectSink(DataStream<RowData> stre
541548
return iterator;
542549
}
543550

544-
private static List<String> convertRowDataToRowString(List<RowData> rows) {
551+
private static String convertRowDataToRowString(RowData row) {
545552
LinkedHashMap<String, Integer> map = new LinkedHashMap<>();
546553
map.put("id", 0);
547554
map.put("name", 1);
548555
map.put("address", 2);
549556
map.put("phone_number", 3);
550557
map.put("_table_name", 4);
551-
return rows.stream()
552-
.map(
553-
row ->
554-
RowUtils.createRowWithNamedPositions(
555-
row.getRowKind(),
556-
new Object[] {
557-
row.getLong(0),
558-
row.getString(1),
559-
row.getString(2),
560-
row.getString(3),
561-
row.getString(4)
562-
},
563-
map)
564-
.toString())
565-
.collect(Collectors.toList());
558+
return RowUtils.createRowWithNamedPositions(
559+
row.getRowKind(),
560+
new Object[] {
561+
row.getLong(0),
562+
row.getString(1),
563+
row.getString(2),
564+
row.getString(3),
565+
row.getString(4)
566+
},
567+
map)
568+
.toString();
566569
}
567570

568571
private void testRemoveTablesOneByOne(

‎flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxCharsetITCase.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,8 @@ public void testCharset() throws Exception {
169169
+ " 'scan.incremental.snapshot.chunk.size' = '%s'"
170170
+ ")",
171171
testName,
172-
HOST_NAME,
173-
PORT,
172+
getHost(),
173+
getPort(),
174174
USER_NAME,
175175
PASSWORD,
176176
DATABASE,
@@ -188,7 +188,7 @@ public void testCharset() throws Exception {
188188
waitForSnapshotStarted(iterator);
189189
assertEqualsInAnyOrder(
190190
Arrays.asList(snapshotExpected),
191-
fetchAndConvert(iterator, snapshotExpected.length, Row::toString));
191+
fetchAndConvert(iterator, snapshotExpected.length, WAITING_TIMEOUT, Row::toString));
192192

193193
// test binlog phase
194194
try (Connection connection = getJdbcConnection();
@@ -200,7 +200,7 @@ public void testCharset() throws Exception {
200200
}
201201
assertEqualsInAnyOrder(
202202
Arrays.asList(binlogExpected),
203-
fetchAndConvert(iterator, binlogExpected.length, Row::toString));
203+
fetchAndConvert(iterator, binlogExpected.length, WAITING_TIMEOUT, Row::toString));
204204
result.getJobClient().get().cancel().get();
205205
}
206206

‎flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxSourceITCase.java

+16-11
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,8 @@ public void testSingleKey() throws Exception {
8484
+ " 'server-time-zone' = 'UTC',"
8585
+ " 'server-id' = '%s'"
8686
+ ")",
87-
HOST_NAME,
88-
PORT,
87+
getHost(),
88+
getPort(),
8989
USER_NAME,
9090
PASSWORD,
9191
DATABASE,
@@ -110,7 +110,8 @@ public void testSingleKey() throws Exception {
110110
}
111111

112112
List<String> realSnapshotData =
113-
fetchAndConvert(iterator, expectedSnapshotData.size(), Row::toString);
113+
fetchAndConvert(
114+
iterator, expectedSnapshotData.size(), WAITING_TIMEOUT, Row::toString);
114115
assertEqualsInAnyOrder(expectedSnapshotData, realSnapshotData);
115116

116117
// second step: check the sink data
@@ -157,7 +158,8 @@ public void testSingleKey() throws Exception {
157158
for (int i = 0; i < captureCustomerTables.length; i++) {
158159
expectedBinlogData.addAll(Arrays.asList(expectedBinlog));
159160
}
160-
List<String> realBinlog = fetchAndConvert(iterator, expectedBinlog.length, Row::toString);
161+
List<String> realBinlog =
162+
fetchAndConvert(iterator, expectedBinlog.length, WAITING_TIMEOUT, Row::toString);
161163
assertEqualsInOrder(expectedBinlogData, realBinlog);
162164
tableResult.getJobClient().get().cancel().get();
163165
}
@@ -236,8 +238,8 @@ public void testFullTypesDdl() throws Exception {
236238
+ " 'server-time-zone' = 'UTC',"
237239
+ " 'server-id' = '%s'"
238240
+ ")",
239-
HOST_NAME,
240-
PORT,
241+
getHost(),
242+
getPort(),
241243
USER_NAME,
242244
PASSWORD,
243245
DATABASE,
@@ -247,7 +249,8 @@ public void testFullTypesDdl() throws Exception {
247249

248250
TableResult tableResult = tEnv.executeSql("select * from polardbx_full_types");
249251
CloseableIterator<Row> iterator = tableResult.collect();
250-
List<String> realSnapshotData = fetchAndConvert(iterator, 1, Row::toString);
252+
List<String> realSnapshotData =
253+
fetchAndConvert(iterator, 1, WAITING_TIMEOUT, Row::toString);
251254
String[] expectedSnapshotData =
252255
new String[] {
253256
"+I[100001, 127, 255, 32767, 65535, 8388607, 16777215, 2147483647, 4294967295, 2147483647, "
@@ -303,8 +306,8 @@ public void testMultiKeys() throws Exception {
303306
+ " 'server-time-zone' = 'UTC',"
304307
+ " 'server-id' = '%s'"
305308
+ ")",
306-
HOST_NAME,
307-
PORT,
309+
getHost(),
310+
getPort(),
308311
USER_NAME,
309312
PASSWORD,
310313
DATABASE,
@@ -329,7 +332,8 @@ public void testMultiKeys() throws Exception {
329332
}
330333

331334
List<String> realSnapshotData =
332-
fetchAndConvert(iterator, expectedSnapshotData.size(), Row::toString);
335+
fetchAndConvert(
336+
iterator, expectedSnapshotData.size(), WAITING_TIMEOUT, Row::toString);
333337
assertEqualsInAnyOrder(expectedSnapshotData, realSnapshotData);
334338

335339
// second step: check the sink data
@@ -378,7 +382,8 @@ public void testMultiKeys() throws Exception {
378382
"+I[7, 9999, 9999, 1007, 2022-01-17T00:00]",
379383
"-D[7, 9999, 9999, 1007, 2022-01-17T00:00]"
380384
};
381-
List<String> realBinlog = fetchAndConvert(iterator, expectedBinlog.length, Row::toString);
385+
List<String> realBinlog =
386+
fetchAndConvert(iterator, expectedBinlog.length, WAITING_TIMEOUT, Row::toString);
382387
assertEqualsInAnyOrder(Arrays.asList(expectedBinlog), realBinlog);
383388
tableResult.getJobClient().get().cancel().get();
384389
}

‎flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/polardbx/PolardbxSourceTestBase.java

+24-25
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,10 @@
1717

1818
package org.apache.flink.cdc.connectors.polardbx;
1919

20+
import org.apache.flink.cdc.common.utils.TestCaseUtils;
2021
import org.apache.flink.table.planner.factories.TestValuesTableFactory;
2122
import org.apache.flink.test.util.AbstractTestBase;
2223

23-
import com.github.dockerjava.api.model.ExposedPort;
24-
import com.github.dockerjava.api.model.PortBinding;
25-
import com.github.dockerjava.api.model.Ports;
2624
import org.apache.commons.lang3.StringUtils;
2725
import org.junit.AfterClass;
2826
import org.junit.BeforeClass;
@@ -61,36 +59,37 @@
6159
public abstract class PolardbxSourceTestBase extends AbstractTestBase {
6260
private static final Logger LOG = LoggerFactory.getLogger(PolardbxSourceTestBase.class);
6361
private static final Pattern COMMENT_PATTERN = Pattern.compile("^(.*)--.*$");
64-
protected static final Integer PORT = 8527;
65-
protected static final String HOST_NAME = "127.0.0.1";
66-
protected static final String USER_NAME = "polardbx_root";
67-
protected static final String PASSWORD = "123456";
62+
6863
private static final String IMAGE_VERSION = "2.1.0";
6964
private static final DockerImageName POLARDBX_IMAGE =
7065
DockerImageName.parse("polardbx/polardb-x:" + IMAGE_VERSION);
7166

67+
protected static final Integer INNER_PORT = 8527;
68+
protected static final String USER_NAME = "polardbx_root";
69+
protected static final String PASSWORD = "123456";
70+
protected static final Duration WAITING_TIMEOUT = Duration.ofMinutes(1);
71+
7272
protected static final GenericContainer POLARDBX_CONTAINER =
7373
new GenericContainer<>(POLARDBX_IMAGE)
74-
.withExposedPorts(PORT)
74+
.withExposedPorts(INNER_PORT)
7575
.withLogConsumer(new Slf4jLogConsumer(LOG))
76-
.withStartupTimeout(Duration.ofMinutes(3))
77-
.withCreateContainerCmdModifier(
78-
c ->
79-
c.withPortBindings(
80-
new PortBinding(
81-
Ports.Binding.bindPort(PORT),
82-
new ExposedPort(PORT))));
76+
.withStartupTimeout(Duration.ofMinutes(3));
77+
78+
protected static String getHost() {
79+
return POLARDBX_CONTAINER.getHost();
80+
}
81+
82+
protected static int getPort() {
83+
return POLARDBX_CONTAINER.getMappedPort(INNER_PORT);
84+
}
8385

8486
@BeforeClass
85-
public static void startContainers() throws InterruptedException {
86-
// no need to start container when the port 8527 is listening
87-
if (!checkConnection()) {
88-
LOG.info("Polardbx connection is not valid, so try to start containers...");
89-
Startables.deepStart(Stream.of(POLARDBX_CONTAINER)).join();
90-
LOG.info("Containers are started.");
91-
// here should wait 10s that make sure the polardbx is ready
92-
Thread.sleep(10 * 1000);
93-
}
87+
public static void startContainers() {
88+
Startables.deepStart(Stream.of(POLARDBX_CONTAINER)).join();
89+
LOG.info("Containers are started.");
90+
91+
TestCaseUtils.repeatedCheck(
92+
PolardbxSourceTestBase::checkConnection, WAITING_TIMEOUT, Duration.ofSeconds(1));
9493
}
9594

9695
@AfterClass
@@ -101,7 +100,7 @@ public static void stopContainers() {
101100
}
102101

103102
protected static String getJdbcUrl() {
104-
return String.format("jdbc:mysql://%s:%s", HOST_NAME, PORT);
103+
return String.format("jdbc:mysql://%s:%s", getHost(), getPort());
105104
}
106105

107106
protected static Connection getJdbcConnection() throws SQLException {

0 commit comments

Comments
 (0)