From 6508049164692aacc3dc7764dc746fa399b55b17 Mon Sep 17 00:00:00 2001 From: FuYouJ <1247908487@qq.com> Date: Sat, 14 Sep 2024 15:37:07 +0800 Subject: [PATCH 01/10] [improve][Redis]Redis scan command supports versions 3, 4, 5, 6, 7 --- docs/en/connector-v2/source/Redis.md | 78 ++++++++++++++----- .../seatunnel/redis/client/RedisClient.java | 34 +++++++- .../seatunnel/redis/config/RedisConfig.java | 6 ++ .../redis/config/RedisParameters.java | 6 ++ .../seatunnel/redis/config/RedisVersion.java | 25 ++++++ .../redis/source/RedisSourceFactory.java | 3 +- .../redis/source/RedisSourceReader.java | 6 +- .../e2e/connector/redis/Redis3IT.java | 33 ++++++++ .../e2e/connector/redis/Redis4IT.java | 33 ++++++++ .../e2e/connector/redis/Redis5IT.java | 32 ++++++++ .../e2e/connector/redis/Redis6IT.java | 33 ++++++++ .../e2e/connector/redis/Redis7IT.java | 33 ++++++++ .../e2e/connector/redis/RedisIT.java | 59 ++++++++------ .../resources/redis-to-redis-by-db-num.conf | 1 + .../test/resources/redis-to-redis-expire.conf | 1 + .../src/test/resources/redis-to-redis.conf | 1 + .../scan-hash-to-redis-list-hash-check.conf | 1 + ...st-test-read-to-redis-list-test-check.conf | 1 + .../scan-set-to-redis-list-set-check.conf | 1 + .../test/resources/scan-string-to-redis.conf | 1 + .../scan-zset-to-redis-list-zset-check.conf | 1 + 21 files changed, 343 insertions(+), 46 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisVersion.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java diff --git a/docs/en/connector-v2/source/Redis.md b/docs/en/connector-v2/source/Redis.md index 9f4e86c8fb7..1ec7dc8f752 100644 --- a/docs/en/connector-v2/source/Redis.md +++ b/docs/en/connector-v2/source/Redis.md @@ -17,8 +17,9 @@ Used to read data from Redis. ## Options -| name | type | required | default value | -|---------------------|--------|-----------------------|---------------| +| name | type | required | default value | +| ------------------- | ------ | --------------------- | ------------- | +| redis_version | string | yes | - | | host | string | yes | - | | port | int | yes | - | | keys | string | yes | - | @@ -34,6 +35,10 @@ Used to read data from Redis. | format | string | no | json | | common-options | | no | - | +### redis_version + +version of redis, support `Redis3`,` Redis4`, `Redis5`,` Redis6`,` Redis7` + ### host [string] redis host @@ -67,7 +72,6 @@ for example, if the value of hash key is the following shown: if hash_key_parse_mode is `all` and schema config as the following shown, it will generate the following data: ```hocon - schema { fields { 001 { @@ -83,14 +87,13 @@ schema { ``` -| 001 | 002 | -|---------------------------------|---------------------------| +| 001 | 002 | +| ------------------------------- | ------------------------- | | Row(name=tyrantlucifer, age=26) | Row(name=Zongwen, age=26) | if hash_key_parse_mode is `kv` and schema config as the following shown, it will generate the following data: ```hocon - schema { fields { hash_key = string @@ -101,10 +104,10 @@ schema { ``` -| hash_key | name | age | -|----------|---------------|-----| -| 001 | tyrantlucifer | 26 | -| 002 | Zongwen | 26 | +| hash_key | name | age | +| -------- | ------------- | ---- | +| 001 | tyrantlucifer | 26 | +| 002 | Zongwen | 26 | each kv that in hash key it will be treated as a row and send it to upstream. @@ -180,7 +183,6 @@ when you assign format is `json`, you should also assign schema option, for exam upstream data is the following: ```json - {"code": 200, "data": "get success", "success": true} ``` @@ -188,7 +190,6 @@ upstream data is the following: you should assign schema as the following: ```hocon - schema { fields { code = int @@ -201,8 +202,8 @@ schema { connector will generate data as the following: -| code | data | success | -|------|-------------|---------| +| code | data | success | +| ---- | ----------- | ------- | | 200 | get success | true | when you assign format is `text`, connector will do nothing for upstream data, for example: @@ -210,15 +211,14 @@ when you assign format is `text`, connector will do nothing for upstream data, f upstream data is the following: ```json - {"code": 200, "data": "get success", "success": true} ``` connector will generate data as the following: -| content | -|----------------------------------------------------------| +| content | +| -------------------------------------------------------- | | {"code": 200, "data": "get success", "success": true} | ### schema [config] @@ -242,6 +242,7 @@ Redis { keys = "key_test*" data_type = key format = text + redis_version = Redis5 } ``` @@ -252,6 +253,7 @@ Redis { keys = "key_test*" data_type = key format = json + redis_version = Redis5 schema { fields { name = string @@ -261,6 +263,46 @@ Redis { } ``` +read string type keys write append to list + +```hocon +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "string_test*" + data_type = string + batch_size = 33 + redis_version = Redis7 + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "string_test_list" + data_type = list + batch_size = 33 + } +} +``` + ## Changelog ### 2.2.0-beta 2022-09-26 @@ -270,4 +312,4 @@ Redis { ### next version - [Improve] Support redis cluster mode connection and user authentication [3188](https://github.com/apache/seatunnel/pull/3188) - +- [Bug] Redis scan command supports versions 3, 4, 5, 6, 7 \ No newline at end of file diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClient.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClient.java index 5730838ccae..a96a221ff31 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClient.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClient.java @@ -19,11 +19,13 @@ import org.apache.seatunnel.connectors.seatunnel.redis.config.RedisDataType; import org.apache.seatunnel.connectors.seatunnel.redis.config.RedisParameters; +import org.apache.seatunnel.connectors.seatunnel.redis.config.RedisVersion; import redis.clients.jedis.Jedis; import redis.clients.jedis.params.ScanParams; import redis.clients.jedis.resps.ScanResult; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -32,6 +34,8 @@ public abstract class RedisClient extends Jedis { protected final RedisParameters redisParameters; + private final RedisVersion redisVersion; + protected final int batchSize; protected final Jedis jedis; @@ -40,6 +44,7 @@ protected RedisClient(RedisParameters redisParameters, Jedis jedis) { this.redisParameters = redisParameters; this.batchSize = redisParameters.getBatchSize(); this.jedis = jedis; + this.redisVersion = redisParameters.getRedisVersion(); } public ScanResult scanKeys( @@ -47,7 +52,34 @@ public ScanResult scanKeys( ScanParams scanParams = new ScanParams(); scanParams.match(keysPattern); scanParams.count(batchSize); - return jedis.scan(cursor, scanParams, type.name()); + return scanByRedisVersion(cursor, scanParams, type, redisVersion); + } + + private ScanResult scanByRedisVersion( + String cursor, ScanParams scanParams, RedisDataType type, RedisVersion redisVersion) { + if (RedisVersion.Redis3.equals(redisVersion) + || RedisVersion.Redis4.equals(redisVersion) + || RedisVersion.Redis5.equals(redisVersion)) { + return scanOnRedis5(cursor, scanParams, type); + } else { + return jedis.scan(cursor, scanParams, type.name()); + } + } + + // When the version is earlier than redis5, scan command does not support type + private ScanResult scanOnRedis5( + String cursor, ScanParams scanParams, RedisDataType type) { + ScanResult scanResult = jedis.scan(cursor, scanParams); + String resultCursor = scanResult.getCursor(); + List keys = scanResult.getResult(); + List typeKeys = new ArrayList<>(keys.size()); + for (String key : keys) { + String keyType = jedis.type(key); + if (type.name().equalsIgnoreCase(keyType)) { + typeKeys.add(key); + } + } + return new ScanResult<>(resultCursor, typeKeys); } public abstract List batchGetString(List keys); diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisConfig.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisConfig.java index 3be5b39de99..a88c0e5726e 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisConfig.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisConfig.java @@ -36,6 +36,12 @@ public enum HashKeyParseMode { KV; } + public static final Option REDIS_VERSION = + Options.key("redis_version") + .enumType(RedisVersion.class) + .noDefaultValue() + .withDescription("the version of redis,minimum supported Redis3"); + public static final Option HOST = Options.key("host") .stringType() diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisParameters.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisParameters.java index ef1fe104d7a..90eaa78ac2c 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisParameters.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisParameters.java @@ -36,6 +36,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Optional; @Data public class RedisParameters implements Serializable { @@ -53,7 +54,12 @@ public class RedisParameters implements Serializable { private long expire = RedisConfig.EXPIRE.defaultValue(); private int batchSize = RedisConfig.BATCH_SIZE.defaultValue(); + private RedisVersion redisVersion; + public void buildWithConfig(ReadonlyConfig config) { + // redis version + Optional versionOptional = config.getOptional(RedisConfig.REDIS_VERSION); + versionOptional.ifPresent(version -> this.redisVersion = version); // set host this.host = config.get(RedisConfig.HOST); // set port diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisVersion.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisVersion.java new file mode 100644 index 00000000000..17b2cb0fa16 --- /dev/null +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisVersion.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.seatunnel.connectors.seatunnel.redis.config; + +public enum RedisVersion { + Redis3, + Redis4, + Redis5, + Redis6, + Redis7; +} diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java index c4f9ac099e9..4fd0403e5a2 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java @@ -51,7 +51,8 @@ public OptionRule optionRule() { RedisConfig.HOST, RedisConfig.PORT, RedisConfig.KEY_PATTERN, - RedisConfig.DATA_TYPE) + RedisConfig.DATA_TYPE, + RedisConfig.REDIS_VERSION) .optional( RedisConfig.MODE, RedisConfig.HASH_KEY_PARSE_MODE, diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceReader.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceReader.java index 10825be8bc6..be67c266f9b 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceReader.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceReader.java @@ -80,9 +80,6 @@ public void internalPollNext(Collector output) throws Exception { redisClient.scanKeys(cursor, batchSize, keysPattern, redisDataType); cursor = scanResult.getCursor(); List keys = scanResult.getResult(); - if (CollectionUtils.isEmpty(keys)) { - break; - } pollNext(keys, redisDataType, output); // when cursor return "0", scan end if (ScanParams.SCAN_POINTER_START.equals(cursor)) { @@ -94,6 +91,9 @@ public void internalPollNext(Collector output) throws Exception { private void pollNext(List keys, RedisDataType dataType, Collector output) throws IOException { + if (CollectionUtils.isEmpty(keys)) { + return; + } if (RedisDataType.HASH.equals(dataType)) { pollHashMapToNext(keys, output); return; diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java new file mode 100644 index 00000000000..9fce4bf62d9 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.seatunnel.e2e.connector.redis; + +import java.util.Collections; +import java.util.List; + +public class Redis3IT extends RedisIT { + + @Override + public List getVariable() { + return Collections.singletonList("redisVersion=Redis3"); + } + + @Override + public String getImage() { + return "redis:3"; + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java new file mode 100644 index 00000000000..42b3a159439 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.seatunnel.e2e.connector.redis; + +import java.util.Collections; +import java.util.List; + +public class Redis4IT extends RedisIT { + + @Override + public List getVariable() { + return Collections.singletonList("redisVersion=Redis4"); + } + + @Override + public String getImage() { + return "redis:4"; + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java new file mode 100644 index 00000000000..351ab60bd92 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.seatunnel.e2e.connector.redis; + +import java.util.Collections; +import java.util.List; + +public class Redis5IT extends RedisIT { + @Override + public List getVariable() { + return Collections.singletonList("redisVersion=Redis5"); + } + + @Override + public String getImage() { + return "redis:5"; + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java new file mode 100644 index 00000000000..e24f8d1307e --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.seatunnel.e2e.connector.redis; + +import java.util.Collections; +import java.util.List; + +public class Redis6IT extends RedisIT { + + @Override + public List getVariable() { + return Collections.singletonList("redisVersion=Redis6"); + } + + @Override + public String getImage() { + return "redis:6"; + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java new file mode 100644 index 00000000000..a89d80de2c5 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.seatunnel.e2e.connector.redis; + +import java.util.Collections; +import java.util.List; + +public class Redis7IT extends RedisIT { + + @Override + public List getVariable() { + return Collections.singletonList("redisVersion=Redis7"); + } + + @Override + public String getImage() { + return "redis:7"; + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java index 280ca94faf1..bb93ed09604 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java @@ -64,13 +64,12 @@ @Slf4j public class RedisIT extends TestSuiteBase implements TestResource { - private static final String IMAGE = "redis:latest"; - private static final String HOST = "redis-e2e"; - private static final int PORT = 6379; - private static final String PASSWORD = "SeaTunnel"; - private static final Pair> TEST_DATASET = - generateTestDataSet(); + private final String host = "redis-e2e"; + private final int port = 6379; + private final String password = "SeaTunnel"; + + private final Pair> testDateSet = generateTestDataSet(); private GenericContainer redisContainer; @@ -80,12 +79,13 @@ public class RedisIT extends TestSuiteBase implements TestResource { @Override public void startUp() { this.redisContainer = - new GenericContainer<>(DockerImageName.parse(IMAGE)) + new GenericContainer<>(DockerImageName.parse(getImage())) .withNetwork(NETWORK) - .withNetworkAliases(HOST) - .withExposedPorts(PORT) - .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))) - .withCommand(String.format("redis-server --requirepass %s", PASSWORD)) + .withNetworkAliases(host) + .withExposedPorts(port) + .withLogConsumer( + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(getImage()))) + .withCommand(String.format("redis-server --requirepass %s", password)) .waitingFor( new HostPortWaitStrategy() .withStartupTimeout(Duration.ofMinutes(2))); @@ -97,8 +97,8 @@ public void startUp() { private void initSourceData() { JsonSerializationSchema jsonSerializationSchema = - new JsonSerializationSchema(TEST_DATASET.getKey()); - List rows = TEST_DATASET.getValue(); + new JsonSerializationSchema(testDateSet.getKey()); + List rows = testDateSet.getValue(); for (int i = 0; i < rows.size(); i++) { jedis.set("key_test" + i, new String(jsonSerializationSchema.serialize(rows.get(i)))); } @@ -177,7 +177,7 @@ private static Pair> generateTestDataSet() private void initJedis() { Jedis jedis = new Jedis(redisContainer.getHost(), redisContainer.getFirstMappedPort()); - jedis.auth(PASSWORD); + jedis.auth(password); jedis.ping(); this.jedis = jedis; } @@ -196,7 +196,8 @@ public void tearDown() { @TestTemplate public void testRedis(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = container.executeJob("/redis-to-redis.conf"); + Container.ExecResult execResult = + container.executeJob("/redis-to-redis.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -207,7 +208,8 @@ public void testRedis(TestContainer container) throws IOException, InterruptedEx @TestTemplate public void testRedisWithExpire(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = container.executeJob("/redis-to-redis-expire.conf"); + Container.ExecResult execResult = + container.executeJob("/redis-to-redis-expire.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -217,7 +219,8 @@ public void testRedisWithExpire(TestContainer container) @TestTemplate public void testRedisDbNum(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = container.executeJob("/redis-to-redis-by-db-num.conf"); + Container.ExecResult execResult = + container.executeJob("/redis-to-redis-by-db-num.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(2); Assertions.assertEquals(100, jedis.llen("db_test")); @@ -232,7 +235,8 @@ public void testScanStringTypeWriteRedis(TestContainer container) for (int i = 0; i < 1000; i++) { jedis.set(keyPrefix + i, "val"); } - Container.ExecResult execResult = container.executeJob("/scan-string-to-redis.conf"); + Container.ExecResult execResult = + container.executeJob("/scan-string-to-redis.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("string_test_list", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -253,7 +257,8 @@ public void testScanListTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob("/scan-list-test-read-to-redis-list-test-check.conf"); + container.executeJob( + "/scan-list-test-read-to-redis-list-test-check.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-test-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -275,7 +280,7 @@ public void testScanSetTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob("/scan-set-to-redis-list-set-check.conf"); + container.executeJob("/scan-set-to-redis-list-set-check.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-set-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -297,7 +302,7 @@ public void testScanHashTypeWriteRedis(TestContainer container) jedis.hset(setKey, map); } Container.ExecResult execResult = - container.executeJob("/scan-hash-to-redis-list-hash-check.conf"); + container.executeJob("/scan-hash-to-redis-list-hash-check.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-hash-check", 0, -1); Assertions.assertEquals(100, list.size()); @@ -325,7 +330,7 @@ public void testScanZsetTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob("/scan-zset-to-redis-list-zset-check.conf"); + container.executeJob("/scan-zset-to-redis-list-zset-check.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-zset-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -344,11 +349,19 @@ public void testScanZsetTypeWriteRedis(TestContainer container) public void testMultipletableRedisSink(TestContainer container) throws IOException, InterruptedException { Container.ExecResult execResult = - container.executeJob("/fake-to-multipletableredissink.conf"); + container.executeJob("/fake-to-multipletableredissink.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(3); Assertions.assertEquals(2, jedis.llen("key_multi_list")); jedis.del("key_multi_list"); jedis.select(0); } + + public List getVariable() { + return Collections.singletonList("redisVersion=Redis7"); + } + + public String getImage() { + return "redis:latest"; + } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf index 9c1dec0c367..c32864baaba 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf @@ -37,6 +37,7 @@ source { data_type = key db_num=1 batch_size = 33 + redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf index 133773d799d..a0c5288c542 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf @@ -36,6 +36,7 @@ source { keys = "key_test*" data_type = key batch_size = 33 + redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf index 20fee261f00..1fbbd3cea18 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf @@ -36,6 +36,7 @@ source { keys = "key_test*" data_type = string batch_size = 33 + redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf index ee76a0a8391..590ec4db0d4 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf @@ -36,6 +36,7 @@ source { keys = "key-test-hash*" data_type = hash batch_size = 33 + redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf index 62db2eaa166..6c03ae9764c 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf @@ -36,6 +36,7 @@ source { keys = "list-test-read*" data_type = list batch_size = 33 + redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf index 4c64224863f..e4c0cfd7616 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf @@ -36,6 +36,7 @@ source { keys = "key-test-set*" data_type = set batch_size = 33 + redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf index 317e603978e..fabbdb8d732 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf @@ -36,6 +36,7 @@ source { keys = "string_test*" data_type = string batch_size = 33 + redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf index f7d2c05d37a..6c24823aabf 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf @@ -36,6 +36,7 @@ source { keys = "key-test-zset*" data_type = zset batch_size = 33 + redis_version = ${redisVersion} } } From 997d793c32f831a3665696f4eb111b779da0bbdc Mon Sep 17 00:00:00 2001 From: FuYouJ <1247908487@qq.com> Date: Sat, 14 Sep 2024 15:45:42 +0800 Subject: [PATCH 02/10] [improve][Redis]Redis scan command supports versions 3, 4, 5, 6, 7 --- docs/en/connector-v2/source/Redis.md | 2 +- release-note.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/en/connector-v2/source/Redis.md b/docs/en/connector-v2/source/Redis.md index 1ec7dc8f752..f7eedb0618f 100644 --- a/docs/en/connector-v2/source/Redis.md +++ b/docs/en/connector-v2/source/Redis.md @@ -312,4 +312,4 @@ sink { ### next version - [Improve] Support redis cluster mode connection and user authentication [3188](https://github.com/apache/seatunnel/pull/3188) -- [Bug] Redis scan command supports versions 3, 4, 5, 6, 7 \ No newline at end of file +- [Bug] Redis scan command supports versions 3, 4, 5, 6, 7 [7666](https://github.com/apache/seatunnel/pull/7666) \ No newline at end of file diff --git a/release-note.md b/release-note.md index 4ed0d51fed9..6147093eee7 100644 --- a/release-note.md +++ b/release-note.md @@ -58,7 +58,7 @@ - [Connector-v2] [Mongodb] Support to convert to double from numeric type that mongodb saved it as numeric internally (#6997) - [Connector-v2] [Redis] Using scan replace keys operation command,support batchWrite in single mode(#7030,#7085) - [Connector-V2] [Clickhouse] Add a new optional configuration `clickhouse.config` to the source connector of ClickHouse (#7143) -- [Connector-V2] [ElasticsSource] Source support multiSource (#6730) +- [Connector-V2] [Redis] Redis scan command supports versions 3, 4, 5, 6, 7 (#7666) ### Zeta(ST-Engine) @@ -200,6 +200,7 @@ - [Connector-V2] [Assert] Support field type assert and field value equality assert for full data types (#6275) - [Connector-V2] [Iceberg] Support iceberg sink #6198 - [Connector-V2] [FILE-OBS] Add Huawei Cloud OBS connector #4578 +- [Connector-V2] [ElasticsSource] Source support multiSource (#6730) ### Zeta(ST-Engine) From 0b3ce6b1562dbe7c42b14e86a7b5aefd4a64f3d2 Mon Sep 17 00:00:00 2001 From: FuYouJ <1247908487@qq.com> Date: Sun, 15 Sep 2024 01:18:41 +0800 Subject: [PATCH 03/10] [improve][Redis]Divide the tests of Redis 3, 4, 5, 6, 7 versions into different maven modules --- .../connector-redis-e2e-common/pom.xml | 58 +++++++++++ .../connector/redis/RedisContainerInfo.java | 47 +++++++++ .../redis/RedisTestCaseTemplateIT.java} | 91 ++++++++++++----- .../fake-to-multipletableredissink.conf | 0 .../resources/redis-to-redis-by-db-num.conf | 0 .../test/resources/redis-to-redis-expire.conf | 0 .../src/test/resources/redis-to-redis.conf | 0 .../scan-hash-to-redis-list-hash-check.conf | 0 ...st-test-read-to-redis-list-test-check.conf | 0 .../scan-set-to-redis-list-set-check.conf | 0 .../test/resources/scan-string-to-redis.conf | 0 .../scan-zset-to-redis-list-zset-check.conf | 0 .../connector-redis-e2e-redis3/pom.xml | 29 ++++++ .../e2e/connector/redis/Redis3IT.java | 10 +- .../fake-to-multipletableredissink.conf | 97 +++++++++++++++++++ .../resources/redis-to-redis-by-db-num.conf | 54 +++++++++++ .../test/resources/redis-to-redis-expire.conf | 52 ++++++++++ .../src/test/resources/redis-to-redis.conf | 52 ++++++++++ .../scan-hash-to-redis-list-hash-check.conf | 52 ++++++++++ ...st-test-read-to-redis-list-test-check.conf | 52 ++++++++++ .../scan-set-to-redis-list-set-check.conf | 52 ++++++++++ .../test/resources/scan-string-to-redis.conf | 52 ++++++++++ .../scan-zset-to-redis-list-zset-check.conf | 52 ++++++++++ .../connector-redis-e2e-redis4/pom.xml | 29 ++++++ .../e2e/connector/redis/Redis4IT.java | 10 +- .../fake-to-multipletableredissink.conf | 97 +++++++++++++++++++ .../resources/redis-to-redis-by-db-num.conf | 54 +++++++++++ .../test/resources/redis-to-redis-expire.conf | 52 ++++++++++ .../src/test/resources/redis-to-redis.conf | 52 ++++++++++ .../scan-hash-to-redis-list-hash-check.conf | 52 ++++++++++ ...st-test-read-to-redis-list-test-check.conf | 52 ++++++++++ .../scan-set-to-redis-list-set-check.conf | 52 ++++++++++ .../test/resources/scan-string-to-redis.conf | 52 ++++++++++ .../scan-zset-to-redis-list-zset-check.conf | 52 ++++++++++ .../connector-redis-e2e-redis5/pom.xml | 30 ++++++ .../e2e/connector/redis/Redis5IT.java | 11 ++- .../fake-to-multipletableredissink.conf | 97 +++++++++++++++++++ .../resources/redis-to-redis-by-db-num.conf | 54 +++++++++++ .../test/resources/redis-to-redis-expire.conf | 52 ++++++++++ .../src/test/resources/redis-to-redis.conf | 52 ++++++++++ .../scan-hash-to-redis-list-hash-check.conf | 52 ++++++++++ ...st-test-read-to-redis-list-test-check.conf | 52 ++++++++++ .../scan-set-to-redis-list-set-check.conf | 52 ++++++++++ .../test/resources/scan-string-to-redis.conf | 52 ++++++++++ .../scan-zset-to-redis-list-zset-check.conf | 52 ++++++++++ .../connector-redis-e2e-redis6/pom.xml | 30 ++++++ .../e2e/connector/redis/Redis6IT.java | 10 +- .../fake-to-multipletableredissink.conf | 97 +++++++++++++++++++ .../resources/redis-to-redis-by-db-num.conf | 54 +++++++++++ .../test/resources/redis-to-redis-expire.conf | 52 ++++++++++ .../src/test/resources/redis-to-redis.conf | 52 ++++++++++ .../scan-hash-to-redis-list-hash-check.conf | 52 ++++++++++ ...st-test-read-to-redis-list-test-check.conf | 52 ++++++++++ .../scan-set-to-redis-list-set-check.conf | 52 ++++++++++ .../test/resources/scan-string-to-redis.conf | 52 ++++++++++ .../scan-zset-to-redis-list-zset-check.conf | 52 ++++++++++ .../connector-redis-e2e-redis7/pom.xml | 30 ++++++ .../e2e/connector/redis/Redis7IT.java | 10 +- .../fake-to-multipletableredissink.conf | 97 +++++++++++++++++++ .../resources/redis-to-redis-by-db-num.conf | 54 +++++++++++ .../test/resources/redis-to-redis-expire.conf | 52 ++++++++++ .../src/test/resources/redis-to-redis.conf | 52 ++++++++++ .../scan-hash-to-redis-list-hash-check.conf | 52 ++++++++++ ...st-test-read-to-redis-list-test-check.conf | 52 ++++++++++ .../scan-set-to-redis-list-set-check.conf | 52 ++++++++++ .../test/resources/scan-string-to-redis.conf | 52 ++++++++++ .../scan-zset-to-redis-list-zset-check.conf | 52 ++++++++++ .../connector-redis-e2e/pom.xml | 20 ++++ 68 files changed, 2941 insertions(+), 49 deletions(-) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisContainerInfo.java rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java => connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java} (82%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-common}/src/test/resources/fake-to-multipletableredissink.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-common}/src/test/resources/redis-to-redis-by-db-num.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-common}/src/test/resources/redis-to-redis-expire.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-common}/src/test/resources/redis-to-redis.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-common}/src/test/resources/scan-hash-to-redis-list-hash-check.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-common}/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-common}/src/test/resources/scan-set-to-redis-list-set-check.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-common}/src/test/resources/scan-string-to-redis.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-common}/src/test/resources/scan-zset-to-redis-list-zset-check.conf (100%) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-redis3}/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java (81%) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/fake-to-multipletableredissink.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-by-db-num.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-expire.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-hash-to-redis-list-hash-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-set-to-redis-list-set-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-string-to-redis.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-zset-to-redis-list-zset-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-redis4}/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java (81%) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/fake-to-multipletableredissink.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-by-db-num.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-expire.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-hash-to-redis-list-hash-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-set-to-redis-list-set-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-string-to-redis.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-zset-to-redis-list-zset-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-redis5}/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java (81%) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/fake-to-multipletableredissink.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-by-db-num.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-expire.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-hash-to-redis-list-hash-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-set-to-redis-list-set-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-string-to-redis.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-zset-to-redis-list-zset-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-redis6}/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java (81%) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/fake-to-multipletableredissink.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-by-db-num.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-expire.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-hash-to-redis-list-hash-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-set-to-redis-list-set-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-string-to-redis.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-zset-to-redis-list-zset-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{ => connector-redis-e2e-redis7}/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java (81%) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/fake-to-multipletableredissink.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-by-db-num.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-expire.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-hash-to-redis-list-hash-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-set-to-redis-list-set-check.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-string-to-redis.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-zset-to-redis-list-zset-check.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml new file mode 100644 index 00000000000..9b928f3d48b --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml @@ -0,0 +1,58 @@ + + + 4.0.0 + + org.apache.seatunnel + connector-redis-e2e + ${revision} + + + connector-redis-e2e-common + SeaTunnel : E2E : Connector V2 : Redis : Common + + + 8 + 8 + UTF-8 + + + + + org.apache.seatunnel + connector-redis + ${project.version} + test + + + + + + + true + src/test/resources + + **/*.* + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + false + + + + + test-jar + + + + + + + + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisContainerInfo.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisContainerInfo.java new file mode 100644 index 00000000000..faa7ca844d4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisContainerInfo.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.seatunnel.e2e.connector.redis; + +public class RedisContainerInfo { + private String host; + private int port; + private String password; + private String imageName; + + public RedisContainerInfo(String host, int port, String password, String imageName) { + this.host = host; + this.port = port; + this.password = password; + this.imageName = imageName; + } + + public String getHost() { + return host; + } + + public int getPort() { + return port; + } + + public String getPassword() { + return password; + } + + public String getImageName() { + return imageName; + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java similarity index 82% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java index bb93ed09604..f3964f966d6 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.seatunnel.e2e.connector.redis; import org.apache.seatunnel.api.table.type.ArrayType; @@ -63,13 +62,15 @@ import java.util.stream.Stream; @Slf4j -public class RedisIT extends TestSuiteBase implements TestResource { +public abstract class RedisTestCaseTemplateIT extends TestSuiteBase implements TestResource { + + private String host; + private int port; + private String password; - private final String host = "redis-e2e"; - private final int port = 6379; - private final String password = "SeaTunnel"; + private String imageName; - private final Pair> testDateSet = generateTestDataSet(); + private Pair> testDateSet; private GenericContainer redisContainer; @@ -78,13 +79,14 @@ public class RedisIT extends TestSuiteBase implements TestResource { @BeforeAll @Override public void startUp() { + initContainerInfo(); this.redisContainer = - new GenericContainer<>(DockerImageName.parse(getImage())) + new GenericContainer<>(DockerImageName.parse(imageName)) .withNetwork(NETWORK) .withNetworkAliases(host) .withExposedPorts(port) .withLogConsumer( - new Slf4jLogConsumer(DockerLoggerFactory.getLogger(getImage()))) + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(imageName))) .withCommand(String.format("redis-server --requirepass %s", password)) .waitingFor( new HostPortWaitStrategy() @@ -95,6 +97,15 @@ public void startUp() { this.initSourceData(); } + private void initContainerInfo() { + RedisContainerInfo redisContainerInfo = getRedisContainerInfo(); + this.host = redisContainerInfo.getHost(); + this.port = redisContainerInfo.getPort(); + this.password = redisContainerInfo.getPassword(); + this.imageName = redisContainerInfo.getImageName(); + this.testDateSet = generateTestDataSet(); + } + private void initSourceData() { JsonSerializationSchema jsonSerializationSchema = new JsonSerializationSchema(testDateSet.getKey()); @@ -111,7 +122,7 @@ private void initSourceData() { jedis.select(0); } - private static Pair> generateTestDataSet() { + protected Pair> generateTestDataSet() { SeaTunnelRowType rowType = new SeaTunnelRowType( new String[] { @@ -196,8 +207,7 @@ public void tearDown() { @TestTemplate public void testRedis(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = - container.executeJob("/redis-to-redis.conf", getVariable()); + Container.ExecResult execResult = container.executeJob(testRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -209,7 +219,7 @@ public void testRedis(TestContainer container) throws IOException, InterruptedEx public void testRedisWithExpire(TestContainer container) throws IOException, InterruptedException { Container.ExecResult execResult = - container.executeJob("/redis-to-redis-expire.conf", getVariable()); + container.executeJob(testRedisWithExpireConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -220,7 +230,7 @@ public void testRedisWithExpire(TestContainer container) @TestTemplate public void testRedisDbNum(TestContainer container) throws IOException, InterruptedException { Container.ExecResult execResult = - container.executeJob("/redis-to-redis-by-db-num.conf", getVariable()); + container.executeJob(testRedisDbNumConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(2); Assertions.assertEquals(100, jedis.llen("db_test")); @@ -236,7 +246,7 @@ public void testScanStringTypeWriteRedis(TestContainer container) jedis.set(keyPrefix + i, "val"); } Container.ExecResult execResult = - container.executeJob("/scan-string-to-redis.conf", getVariable()); + container.executeJob(testScanStringTypeWriteRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("string_test_list", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -257,8 +267,7 @@ public void testScanListTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob( - "/scan-list-test-read-to-redis-list-test-check.conf", getVariable()); + container.executeJob(testScanListTypeWriteRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-test-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -280,7 +289,7 @@ public void testScanSetTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob("/scan-set-to-redis-list-set-check.conf", getVariable()); + container.executeJob(testScanSetTypeWriteRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-set-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -302,7 +311,7 @@ public void testScanHashTypeWriteRedis(TestContainer container) jedis.hset(setKey, map); } Container.ExecResult execResult = - container.executeJob("/scan-hash-to-redis-list-hash-check.conf", getVariable()); + container.executeJob(testScanHashTypeWriteRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-hash-check", 0, -1); Assertions.assertEquals(100, list.size()); @@ -330,7 +339,7 @@ public void testScanZsetTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob("/scan-zset-to-redis-list-zset-check.conf", getVariable()); + container.executeJob(testScanZsetTypeWriteRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-zset-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -349,7 +358,7 @@ public void testScanZsetTypeWriteRedis(TestContainer container) public void testMultipletableRedisSink(TestContainer container) throws IOException, InterruptedException { Container.ExecResult execResult = - container.executeJob("/fake-to-multipletableredissink.conf", getVariable()); + container.executeJob(testMultipletableRedisSinkConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(3); Assertions.assertEquals(2, jedis.llen("key_multi_list")); @@ -357,11 +366,45 @@ public void testMultipletableRedisSink(TestContainer container) jedis.select(0); } - public List getVariable() { - return Collections.singletonList("redisVersion=Redis7"); + ///// need different redis version test override + + public String testRedisConf() { + return "/redis-to-redis.conf"; + } + + public String testRedisWithExpireConf() { + return "/redis-to-redis-expire.conf"; + } + + public String testRedisDbNumConf() { + return "/redis-to-redis-by-db-num.conf"; + } + + public String testScanStringTypeWriteRedisConf() { + return "/scan-string-to-redis.conf"; + } + + public String testScanListTypeWriteRedisConf() { + return "/scan-list-test-read-to-redis-list-test-check.conf"; } - public String getImage() { - return "redis:latest"; + public String testScanHashTypeWriteRedisConf() { + return "/scan-hash-to-redis-list-hash-check.conf"; } + + public String testScanZsetTypeWriteRedisConf() { + return "/scan-zset-to-redis-list-zset-check.conf"; + } + + public String testScanSetTypeWriteRedisConf() { + return "/scan-set-to-redis-list-set-check.conf"; + } + + public String testMultipletableRedisSinkConf() { + return "/fake-to-multipletableredissink.conf"; + } + + public abstract RedisContainerInfo getRedisContainerInfo(); + + public abstract List getVariables(); } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/fake-to-multipletableredissink.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/fake-to-multipletableredissink.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/fake-to-multipletableredissink.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis-by-db-num.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis-by-db-num.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis-expire.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis-expire.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-hash-to-redis-list-hash-check.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-hash-to-redis-list-hash-check.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-set-to-redis-list-set-check.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-set-to-redis-list-set-check.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-string-to-redis.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-string-to-redis.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-zset-to-redis-list-zset-check.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-zset-to-redis-list-zset-check.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml new file mode 100644 index 00000000000..754ee0c32c1 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml @@ -0,0 +1,29 @@ + + + 4.0.0 + + org.apache.seatunnel + connector-redis-e2e + ${revision} + + + connector-redis-e2e-redis3 + SeaTunnel : E2E : Connector V2 : Redis : Version3 + + + 8 + 8 + UTF-8 + + + + org.apache.seatunnel + connector-redis-e2e-common + ${revision} + test-jar + test + + + + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java similarity index 81% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java index 9fce4bf62d9..96cd71cd262 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java @@ -19,15 +19,15 @@ import java.util.Collections; import java.util.List; -public class Redis3IT extends RedisIT { +public class Redis3IT extends RedisTestCaseTemplateIT { @Override - public List getVariable() { - return Collections.singletonList("redisVersion=Redis3"); + public RedisContainerInfo getRedisContainerInfo() { + return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:3"); } @Override - public String getImage() { - return "redis:3"; + public List getVariables() { + return Collections.singletonList("redisVersion=Redis3"); } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/fake-to-multipletableredissink.conf new file mode 100644 index 00000000000..051c51eea66 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/fake-to-multipletableredissink.conf @@ -0,0 +1,97 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} +source { + FakeSource { + tables_configs = [ + { + schema = { + table = "redis_sink_1" + fields { + id = int + val_bool = boolean + val_int8 = tinyint + val_int16 = smallint + val_int32 = int + val_int64 = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + val_unixtime_micros = timestamp + } + } + rows = [ + { + kind = INSERT + fields = [1, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] + } + ] + }, + { + schema = { + table = "redis_sink_2" + fields { + id = int + val_bool = boolean + val_int8 = tinyint + val_int16 = smallint + val_int32 = int + val_int64 = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + val_unixtime_micros = timestamp + } + } + rows = [ + { + kind = INSERT + fields = [2, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] + } + ] + } + ] + } +} + + +sink { + Redis { + host = "redis-e2e" + port = 6379 + db_num=3 + auth = "U2VhVHVubmVs" + key = "key_multi_list" + data_type = list + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-by-db-num.conf new file mode 100644 index 00000000000..c32864baaba --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-by-db-num.conf @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = key + db_num=1 + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "db_test" + data_type = list + db_num=2 + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-expire.conf new file mode 100644 index 00000000000..a0c5288c542 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-expire.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = key + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "key_list" + data_type = list + expire = 30 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis.conf new file mode 100644 index 00000000000..1fbbd3cea18 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = string + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "key_list" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-hash-to-redis-list-hash-check.conf new file mode 100644 index 00000000000..590ec4db0d4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-hash-to-redis-list-hash-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-hash*" + data_type = hash + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-hash-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf new file mode 100644 index 00000000000..6c03ae9764c --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "list-test-read*" + data_type = list + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-test-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-set-to-redis-list-set-check.conf new file mode 100644 index 00000000000..e4c0cfd7616 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-set-to-redis-list-set-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-set*" + data_type = set + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-set-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-string-to-redis.conf new file mode 100644 index 00000000000..fabbdb8d732 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-string-to-redis.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "string_test*" + data_type = string + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "string_test_list" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-zset-to-redis-list-zset-check.conf new file mode 100644 index 00000000000..6c24823aabf --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-zset-to-redis-list-zset-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-zset*" + data_type = zset + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-zset-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml new file mode 100644 index 00000000000..be93d314104 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml @@ -0,0 +1,29 @@ + + + 4.0.0 + + org.apache.seatunnel + connector-redis-e2e + ${revision} + + + connector-redis-e2e-redis4 + SeaTunnel : E2E : Connector V2 : Redis : Version4 + + + 8 + 8 + UTF-8 + + + + org.apache.seatunnel + connector-redis-e2e-common + ${revision} + test-jar + test + + + + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java similarity index 81% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java index 42b3a159439..dd6bd3ee6c2 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java @@ -19,15 +19,15 @@ import java.util.Collections; import java.util.List; -public class Redis4IT extends RedisIT { +public class Redis4IT extends RedisTestCaseTemplateIT { @Override - public List getVariable() { - return Collections.singletonList("redisVersion=Redis4"); + public RedisContainerInfo getRedisContainerInfo() { + return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:4"); } @Override - public String getImage() { - return "redis:4"; + public List getVariables() { + return Collections.singletonList("redisVersion=Redis4"); } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/fake-to-multipletableredissink.conf new file mode 100644 index 00000000000..051c51eea66 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/fake-to-multipletableredissink.conf @@ -0,0 +1,97 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} +source { + FakeSource { + tables_configs = [ + { + schema = { + table = "redis_sink_1" + fields { + id = int + val_bool = boolean + val_int8 = tinyint + val_int16 = smallint + val_int32 = int + val_int64 = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + val_unixtime_micros = timestamp + } + } + rows = [ + { + kind = INSERT + fields = [1, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] + } + ] + }, + { + schema = { + table = "redis_sink_2" + fields { + id = int + val_bool = boolean + val_int8 = tinyint + val_int16 = smallint + val_int32 = int + val_int64 = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + val_unixtime_micros = timestamp + } + } + rows = [ + { + kind = INSERT + fields = [2, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] + } + ] + } + ] + } +} + + +sink { + Redis { + host = "redis-e2e" + port = 6379 + db_num=3 + auth = "U2VhVHVubmVs" + key = "key_multi_list" + data_type = list + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-by-db-num.conf new file mode 100644 index 00000000000..c32864baaba --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-by-db-num.conf @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = key + db_num=1 + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "db_test" + data_type = list + db_num=2 + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-expire.conf new file mode 100644 index 00000000000..a0c5288c542 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-expire.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = key + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "key_list" + data_type = list + expire = 30 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis.conf new file mode 100644 index 00000000000..1fbbd3cea18 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = string + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "key_list" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-hash-to-redis-list-hash-check.conf new file mode 100644 index 00000000000..590ec4db0d4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-hash-to-redis-list-hash-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-hash*" + data_type = hash + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-hash-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf new file mode 100644 index 00000000000..6c03ae9764c --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "list-test-read*" + data_type = list + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-test-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-set-to-redis-list-set-check.conf new file mode 100644 index 00000000000..e4c0cfd7616 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-set-to-redis-list-set-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-set*" + data_type = set + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-set-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-string-to-redis.conf new file mode 100644 index 00000000000..fabbdb8d732 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-string-to-redis.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "string_test*" + data_type = string + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "string_test_list" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-zset-to-redis-list-zset-check.conf new file mode 100644 index 00000000000..6c24823aabf --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-zset-to-redis-list-zset-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-zset*" + data_type = zset + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-zset-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml new file mode 100644 index 00000000000..b93195e5557 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + org.apache.seatunnel + connector-redis-e2e + ${revision} + + + connector-redis-e2e-redis5 + + SeaTunnel : E2E : Connector V2 : Redis : Version5 + + + 8 + 8 + UTF-8 + + + + org.apache.seatunnel + connector-redis-e2e-common + ${revision} + test-jar + test + + + + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java similarity index 81% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java index 351ab60bd92..347dedb4cfe 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java @@ -19,14 +19,15 @@ import java.util.Collections; import java.util.List; -public class Redis5IT extends RedisIT { +public class Redis5IT extends RedisTestCaseTemplateIT { + @Override - public List getVariable() { - return Collections.singletonList("redisVersion=Redis5"); + public RedisContainerInfo getRedisContainerInfo() { + return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:5"); } @Override - public String getImage() { - return "redis:5"; + public List getVariables() { + return Collections.singletonList("redisVersion=Redis5"); } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/fake-to-multipletableredissink.conf new file mode 100644 index 00000000000..051c51eea66 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/fake-to-multipletableredissink.conf @@ -0,0 +1,97 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} +source { + FakeSource { + tables_configs = [ + { + schema = { + table = "redis_sink_1" + fields { + id = int + val_bool = boolean + val_int8 = tinyint + val_int16 = smallint + val_int32 = int + val_int64 = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + val_unixtime_micros = timestamp + } + } + rows = [ + { + kind = INSERT + fields = [1, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] + } + ] + }, + { + schema = { + table = "redis_sink_2" + fields { + id = int + val_bool = boolean + val_int8 = tinyint + val_int16 = smallint + val_int32 = int + val_int64 = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + val_unixtime_micros = timestamp + } + } + rows = [ + { + kind = INSERT + fields = [2, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] + } + ] + } + ] + } +} + + +sink { + Redis { + host = "redis-e2e" + port = 6379 + db_num=3 + auth = "U2VhVHVubmVs" + key = "key_multi_list" + data_type = list + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-by-db-num.conf new file mode 100644 index 00000000000..c32864baaba --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-by-db-num.conf @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = key + db_num=1 + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "db_test" + data_type = list + db_num=2 + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-expire.conf new file mode 100644 index 00000000000..a0c5288c542 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-expire.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = key + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "key_list" + data_type = list + expire = 30 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis.conf new file mode 100644 index 00000000000..1fbbd3cea18 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = string + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "key_list" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-hash-to-redis-list-hash-check.conf new file mode 100644 index 00000000000..590ec4db0d4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-hash-to-redis-list-hash-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-hash*" + data_type = hash + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-hash-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf new file mode 100644 index 00000000000..6c03ae9764c --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "list-test-read*" + data_type = list + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-test-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-set-to-redis-list-set-check.conf new file mode 100644 index 00000000000..e4c0cfd7616 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-set-to-redis-list-set-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-set*" + data_type = set + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-set-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-string-to-redis.conf new file mode 100644 index 00000000000..fabbdb8d732 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-string-to-redis.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "string_test*" + data_type = string + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "string_test_list" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-zset-to-redis-list-zset-check.conf new file mode 100644 index 00000000000..6c24823aabf --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-zset-to-redis-list-zset-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-zset*" + data_type = zset + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-zset-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml new file mode 100644 index 00000000000..45e4e0ce18b --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + org.apache.seatunnel + connector-redis-e2e + ${revision} + + + connector-redis-e2e-redis6 + + SeaTunnel : E2E : Connector V2 : Redis : Version6 + + + 8 + 8 + UTF-8 + + + + org.apache.seatunnel + connector-redis-e2e-common + ${revision} + test-jar + test + + + + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java similarity index 81% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java index e24f8d1307e..6c5a0295f99 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java @@ -19,15 +19,15 @@ import java.util.Collections; import java.util.List; -public class Redis6IT extends RedisIT { +public class Redis6IT extends RedisTestCaseTemplateIT { @Override - public List getVariable() { - return Collections.singletonList("redisVersion=Redis6"); + public RedisContainerInfo getRedisContainerInfo() { + return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:6"); } @Override - public String getImage() { - return "redis:6"; + public List getVariables() { + return Collections.singletonList("redisVersion=Redis6"); } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/fake-to-multipletableredissink.conf new file mode 100644 index 00000000000..051c51eea66 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/fake-to-multipletableredissink.conf @@ -0,0 +1,97 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} +source { + FakeSource { + tables_configs = [ + { + schema = { + table = "redis_sink_1" + fields { + id = int + val_bool = boolean + val_int8 = tinyint + val_int16 = smallint + val_int32 = int + val_int64 = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + val_unixtime_micros = timestamp + } + } + rows = [ + { + kind = INSERT + fields = [1, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] + } + ] + }, + { + schema = { + table = "redis_sink_2" + fields { + id = int + val_bool = boolean + val_int8 = tinyint + val_int16 = smallint + val_int32 = int + val_int64 = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + val_unixtime_micros = timestamp + } + } + rows = [ + { + kind = INSERT + fields = [2, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] + } + ] + } + ] + } +} + + +sink { + Redis { + host = "redis-e2e" + port = 6379 + db_num=3 + auth = "U2VhVHVubmVs" + key = "key_multi_list" + data_type = list + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-by-db-num.conf new file mode 100644 index 00000000000..c32864baaba --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-by-db-num.conf @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = key + db_num=1 + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "db_test" + data_type = list + db_num=2 + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-expire.conf new file mode 100644 index 00000000000..a0c5288c542 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-expire.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = key + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "key_list" + data_type = list + expire = 30 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis.conf new file mode 100644 index 00000000000..1fbbd3cea18 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = string + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "key_list" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-hash-to-redis-list-hash-check.conf new file mode 100644 index 00000000000..590ec4db0d4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-hash-to-redis-list-hash-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-hash*" + data_type = hash + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-hash-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf new file mode 100644 index 00000000000..6c03ae9764c --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "list-test-read*" + data_type = list + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-test-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-set-to-redis-list-set-check.conf new file mode 100644 index 00000000000..e4c0cfd7616 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-set-to-redis-list-set-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-set*" + data_type = set + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-set-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-string-to-redis.conf new file mode 100644 index 00000000000..fabbdb8d732 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-string-to-redis.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "string_test*" + data_type = string + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "string_test_list" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-zset-to-redis-list-zset-check.conf new file mode 100644 index 00000000000..6c24823aabf --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-zset-to-redis-list-zset-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-zset*" + data_type = zset + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-zset-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml new file mode 100644 index 00000000000..d74a4855820 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + org.apache.seatunnel + connector-redis-e2e + ${revision} + + + connector-redis-e2e-redis7 + + SeaTunnel : E2E : Connector V2 : Redis : Version7 + + + 8 + 8 + UTF-8 + + + + org.apache.seatunnel + connector-redis-e2e-common + ${revision} + test-jar + test + + + + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java similarity index 81% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java index a89d80de2c5..45616620c72 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java @@ -19,15 +19,15 @@ import java.util.Collections; import java.util.List; -public class Redis7IT extends RedisIT { +public class Redis7IT extends RedisTestCaseTemplateIT { @Override - public List getVariable() { - return Collections.singletonList("redisVersion=Redis7"); + public RedisContainerInfo getRedisContainerInfo() { + return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:7"); } @Override - public String getImage() { - return "redis:7"; + public List getVariables() { + return Collections.singletonList("redisVersion=Redis7"); } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/fake-to-multipletableredissink.conf new file mode 100644 index 00000000000..051c51eea66 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/fake-to-multipletableredissink.conf @@ -0,0 +1,97 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +###### +###### This config file is a demonstration of streaming processing in seatunnel config +###### +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} +source { + FakeSource { + tables_configs = [ + { + schema = { + table = "redis_sink_1" + fields { + id = int + val_bool = boolean + val_int8 = tinyint + val_int16 = smallint + val_int32 = int + val_int64 = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + val_unixtime_micros = timestamp + } + } + rows = [ + { + kind = INSERT + fields = [1, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] + } + ] + }, + { + schema = { + table = "redis_sink_2" + fields { + id = int + val_bool = boolean + val_int8 = tinyint + val_int16 = smallint + val_int32 = int + val_int64 = bigint + val_float = float + val_double = double + val_decimal = "decimal(16, 1)" + val_string = string + val_unixtime_micros = timestamp + } + } + rows = [ + { + kind = INSERT + fields = [2, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] + } + ] + } + ] + } +} + + +sink { + Redis { + host = "redis-e2e" + port = 6379 + db_num=3 + auth = "U2VhVHVubmVs" + key = "key_multi_list" + data_type = list + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-by-db-num.conf new file mode 100644 index 00000000000..c32864baaba --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-by-db-num.conf @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = key + db_num=1 + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "db_test" + data_type = list + db_num=2 + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-expire.conf new file mode 100644 index 00000000000..a0c5288c542 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-expire.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = key + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "key_list" + data_type = list + expire = 30 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis.conf new file mode 100644 index 00000000000..1fbbd3cea18 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = string + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "key_list" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-hash-to-redis-list-hash-check.conf new file mode 100644 index 00000000000..590ec4db0d4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-hash-to-redis-list-hash-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-hash*" + data_type = hash + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-hash-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf new file mode 100644 index 00000000000..6c03ae9764c --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "list-test-read*" + data_type = list + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-test-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-set-to-redis-list-set-check.conf new file mode 100644 index 00000000000..e4c0cfd7616 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-set-to-redis-list-set-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-set*" + data_type = set + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-set-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-string-to-redis.conf new file mode 100644 index 00000000000..fabbdb8d732 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-string-to-redis.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "string_test*" + data_type = string + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "string_test_list" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-zset-to-redis-list-zset-check.conf new file mode 100644 index 00000000000..6c24823aabf --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-zset-to-redis-list-zset-check.conf @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" + + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local +} + +source { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key-test-zset*" + data_type = zset + batch_size = 33 + redis_version = ${redisVersion} + } +} + +sink { + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "list-zset-check" + data_type = list + batch_size = 33 + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/pom.xml index 5f3283e805a..042e6eb889c 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/pom.xml @@ -23,8 +23,28 @@ connector-redis-e2e + pom SeaTunnel : E2E : Connector V2 : Redis + + connector-redis-e2e-common + connector-redis-e2e-redis3 + connector-redis-e2e-redis4 + connector-redis-e2e-redis5 + connector-redis-e2e-redis6 + connector-redis-e2e-redis7 + + + + + org.apache.seatunnel + connector-redis + ${project.version} + pom + import + + + From 4725c5786cf540bde14f96fc621dc49619855334 Mon Sep 17 00:00:00 2001 From: FuYouJ <1247908487@qq.com> Date: Sun, 15 Sep 2024 08:43:31 +0800 Subject: [PATCH 04/10] [improve][Redis]Divide the tests of Redis 3, 4, 5, 6, 7 versions into different maven modules,pom add license header --- .../connector-redis-e2e-common/pom.xml | 14 ++++++++++++++ .../connector-redis-e2e-redis3/pom.xml | 14 ++++++++++++++ .../connector-redis-e2e-redis4/pom.xml | 14 ++++++++++++++ .../connector-redis-e2e-redis5/pom.xml | 14 ++++++++++++++ .../connector-redis-e2e-redis6/pom.xml | 14 ++++++++++++++ .../connector-redis-e2e-redis7/pom.xml | 14 ++++++++++++++ 6 files changed, 84 insertions(+) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml index 9b928f3d48b..4692cfd894e 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml @@ -1,4 +1,18 @@ + 4.0.0 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml index 754ee0c32c1..648deebf83f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml @@ -1,4 +1,18 @@ + 4.0.0 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml index be93d314104..11cc66631bd 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml @@ -1,4 +1,18 @@ + 4.0.0 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml index b93195e5557..6d21e887562 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml @@ -1,4 +1,18 @@ + 4.0.0 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml index 45e4e0ce18b..dbb31354823 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml @@ -1,4 +1,18 @@ + 4.0.0 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml index d74a4855820..73f6a12367a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml @@ -1,4 +1,18 @@ + 4.0.0 From 276124601cd7268f438924a3cfa3d0467d7eea85 Mon Sep 17 00:00:00 2001 From: FuYouJ <1247908487@qq.com> Date: Wed, 18 Sep 2024 23:33:35 +0800 Subject: [PATCH 05/10] Revert "[improve][Redis]Divide the tests of Redis 3, 4, 5, 6, 7 versions into different maven modules,pom add license header" This reverts commit 4725c5786cf540bde14f96fc621dc49619855334. --- .../connector-redis-e2e-common/pom.xml | 14 -------------- .../connector-redis-e2e-redis3/pom.xml | 14 -------------- .../connector-redis-e2e-redis4/pom.xml | 14 -------------- .../connector-redis-e2e-redis5/pom.xml | 14 -------------- .../connector-redis-e2e-redis6/pom.xml | 14 -------------- .../connector-redis-e2e-redis7/pom.xml | 14 -------------- 6 files changed, 84 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml index 4692cfd894e..9b928f3d48b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml @@ -1,18 +1,4 @@ - 4.0.0 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml index 648deebf83f..754ee0c32c1 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml @@ -1,18 +1,4 @@ - 4.0.0 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml index 11cc66631bd..be93d314104 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml @@ -1,18 +1,4 @@ - 4.0.0 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml index 6d21e887562..b93195e5557 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml @@ -1,18 +1,4 @@ - 4.0.0 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml index dbb31354823..45e4e0ce18b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml @@ -1,18 +1,4 @@ - 4.0.0 diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml index 73f6a12367a..d74a4855820 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml @@ -1,18 +1,4 @@ - 4.0.0 From f038afd0dffdb5426274d89be4383ebeafa229b3 Mon Sep 17 00:00:00 2001 From: FuYouJ <1247908487@qq.com> Date: Wed, 18 Sep 2024 23:33:35 +0800 Subject: [PATCH 06/10] Revert "[improve][Redis]Divide the tests of Redis 3, 4, 5, 6, 7 versions into different maven modules" This reverts commit 0b3ce6b1562dbe7c42b14e86a7b5aefd4a64f3d2. --- .../connector-redis-e2e-common/pom.xml | 58 ----------- .../connector/redis/RedisContainerInfo.java | 47 --------- .../connector-redis-e2e-redis3/pom.xml | 29 ------ .../fake-to-multipletableredissink.conf | 97 ------------------- .../resources/redis-to-redis-by-db-num.conf | 54 ----------- .../test/resources/redis-to-redis-expire.conf | 52 ---------- .../src/test/resources/redis-to-redis.conf | 52 ---------- .../scan-hash-to-redis-list-hash-check.conf | 52 ---------- ...st-test-read-to-redis-list-test-check.conf | 52 ---------- .../scan-set-to-redis-list-set-check.conf | 52 ---------- .../test/resources/scan-string-to-redis.conf | 52 ---------- .../scan-zset-to-redis-list-zset-check.conf | 52 ---------- .../connector-redis-e2e-redis4/pom.xml | 29 ------ .../fake-to-multipletableredissink.conf | 97 ------------------- .../resources/redis-to-redis-by-db-num.conf | 54 ----------- .../test/resources/redis-to-redis-expire.conf | 52 ---------- .../src/test/resources/redis-to-redis.conf | 52 ---------- .../scan-hash-to-redis-list-hash-check.conf | 52 ---------- ...st-test-read-to-redis-list-test-check.conf | 52 ---------- .../scan-set-to-redis-list-set-check.conf | 52 ---------- .../test/resources/scan-string-to-redis.conf | 52 ---------- .../scan-zset-to-redis-list-zset-check.conf | 52 ---------- .../connector-redis-e2e-redis5/pom.xml | 30 ------ .../fake-to-multipletableredissink.conf | 97 ------------------- .../resources/redis-to-redis-by-db-num.conf | 54 ----------- .../test/resources/redis-to-redis-expire.conf | 52 ---------- .../src/test/resources/redis-to-redis.conf | 52 ---------- .../scan-hash-to-redis-list-hash-check.conf | 52 ---------- ...st-test-read-to-redis-list-test-check.conf | 52 ---------- .../scan-set-to-redis-list-set-check.conf | 52 ---------- .../test/resources/scan-string-to-redis.conf | 52 ---------- .../scan-zset-to-redis-list-zset-check.conf | 52 ---------- .../connector-redis-e2e-redis6/pom.xml | 30 ------ .../fake-to-multipletableredissink.conf | 97 ------------------- .../resources/redis-to-redis-by-db-num.conf | 54 ----------- .../test/resources/redis-to-redis-expire.conf | 52 ---------- .../src/test/resources/redis-to-redis.conf | 52 ---------- .../scan-hash-to-redis-list-hash-check.conf | 52 ---------- ...st-test-read-to-redis-list-test-check.conf | 52 ---------- .../scan-set-to-redis-list-set-check.conf | 52 ---------- .../test/resources/scan-string-to-redis.conf | 52 ---------- .../scan-zset-to-redis-list-zset-check.conf | 52 ---------- .../connector-redis-e2e-redis7/pom.xml | 30 ------ .../fake-to-multipletableredissink.conf | 97 ------------------- .../resources/redis-to-redis-by-db-num.conf | 54 ----------- .../test/resources/redis-to-redis-expire.conf | 52 ---------- .../src/test/resources/redis-to-redis.conf | 52 ---------- .../scan-hash-to-redis-list-hash-check.conf | 52 ---------- ...st-test-read-to-redis-list-test-check.conf | 52 ---------- .../scan-set-to-redis-list-set-check.conf | 52 ---------- .../test/resources/scan-string-to-redis.conf | 52 ---------- .../scan-zset-to-redis-list-zset-check.conf | 52 ---------- .../connector-redis-e2e/pom.xml | 20 ---- .../e2e/connector/redis/Redis3IT.java | 10 +- .../e2e/connector/redis/Redis4IT.java | 10 +- .../e2e/connector/redis/Redis5IT.java | 11 +-- .../e2e/connector/redis/Redis6IT.java | 10 +- .../e2e/connector/redis/Redis7IT.java | 10 +- .../e2e/connector/redis/RedisIT.java} | 91 +++++------------ .../fake-to-multipletableredissink.conf | 0 .../resources/redis-to-redis-by-db-num.conf | 0 .../test/resources/redis-to-redis-expire.conf | 0 .../src/test/resources/redis-to-redis.conf | 0 .../scan-hash-to-redis-list-hash-check.conf | 0 ...st-test-read-to-redis-list-test-check.conf | 0 .../scan-set-to-redis-list-set-check.conf | 0 .../test/resources/scan-string-to-redis.conf | 0 .../scan-zset-to-redis-list-zset-check.conf | 0 68 files changed, 49 insertions(+), 2941 deletions(-) delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisContainerInfo.java delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/fake-to-multipletableredissink.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-by-db-num.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-expire.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-hash-to-redis-list-hash-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-set-to-redis-list-set-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-string-to-redis.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-zset-to-redis-list-zset-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/fake-to-multipletableredissink.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-by-db-num.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-expire.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-hash-to-redis-list-hash-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-set-to-redis-list-set-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-string-to-redis.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-zset-to-redis-list-zset-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/fake-to-multipletableredissink.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-by-db-num.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-expire.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-hash-to-redis-list-hash-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-set-to-redis-list-set-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-string-to-redis.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-zset-to-redis-list-zset-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/fake-to-multipletableredissink.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-by-db-num.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-expire.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-hash-to-redis-list-hash-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-set-to-redis-list-set-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-string-to-redis.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-zset-to-redis-list-zset-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/fake-to-multipletableredissink.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-by-db-num.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-expire.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-hash-to-redis-list-hash-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-set-to-redis-list-set-check.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-string-to-redis.conf delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-zset-to-redis-list-zset-check.conf rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-redis3 => }/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java (81%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-redis4 => }/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java (81%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-redis5 => }/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java (81%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-redis6 => }/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java (81%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-redis7 => }/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java (81%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java => src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java} (82%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-common => }/src/test/resources/fake-to-multipletableredissink.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-common => }/src/test/resources/redis-to-redis-by-db-num.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-common => }/src/test/resources/redis-to-redis-expire.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-common => }/src/test/resources/redis-to-redis.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-common => }/src/test/resources/scan-hash-to-redis-list-hash-check.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-common => }/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-common => }/src/test/resources/scan-set-to-redis-list-set-check.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-common => }/src/test/resources/scan-string-to-redis.conf (100%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/{connector-redis-e2e-common => }/src/test/resources/scan-zset-to-redis-list-zset-check.conf (100%) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml deleted file mode 100644 index 9b928f3d48b..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - 4.0.0 - - org.apache.seatunnel - connector-redis-e2e - ${revision} - - - connector-redis-e2e-common - SeaTunnel : E2E : Connector V2 : Redis : Common - - - 8 - 8 - UTF-8 - - - - - org.apache.seatunnel - connector-redis - ${project.version} - test - - - - - - - true - src/test/resources - - **/*.* - - - - - - org.apache.maven.plugins - maven-jar-plugin - ${maven-jar-plugin.version} - - false - - - - - test-jar - - - - - - - - diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisContainerInfo.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisContainerInfo.java deleted file mode 100644 index faa7ca844d4..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisContainerInfo.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.seatunnel.e2e.connector.redis; - -public class RedisContainerInfo { - private String host; - private int port; - private String password; - private String imageName; - - public RedisContainerInfo(String host, int port, String password, String imageName) { - this.host = host; - this.port = port; - this.password = password; - this.imageName = imageName; - } - - public String getHost() { - return host; - } - - public int getPort() { - return port; - } - - public String getPassword() { - return password; - } - - public String getImageName() { - return imageName; - } -} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml deleted file mode 100644 index 754ee0c32c1..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/pom.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - 4.0.0 - - org.apache.seatunnel - connector-redis-e2e - ${revision} - - - connector-redis-e2e-redis3 - SeaTunnel : E2E : Connector V2 : Redis : Version3 - - - 8 - 8 - UTF-8 - - - - org.apache.seatunnel - connector-redis-e2e-common - ${revision} - test-jar - test - - - - diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/fake-to-multipletableredissink.conf deleted file mode 100644 index 051c51eea66..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/fake-to-multipletableredissink.conf +++ /dev/null @@ -1,97 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -###### -###### This config file is a demonstration of streaming processing in seatunnel config -###### -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} -source { - FakeSource { - tables_configs = [ - { - schema = { - table = "redis_sink_1" - fields { - id = int - val_bool = boolean - val_int8 = tinyint - val_int16 = smallint - val_int32 = int - val_int64 = bigint - val_float = float - val_double = double - val_decimal = "decimal(16, 1)" - val_string = string - val_unixtime_micros = timestamp - } - } - rows = [ - { - kind = INSERT - fields = [1, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] - } - ] - }, - { - schema = { - table = "redis_sink_2" - fields { - id = int - val_bool = boolean - val_int8 = tinyint - val_int16 = smallint - val_int32 = int - val_int64 = bigint - val_float = float - val_double = double - val_decimal = "decimal(16, 1)" - val_string = string - val_unixtime_micros = timestamp - } - } - rows = [ - { - kind = INSERT - fields = [2, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] - } - ] - } - ] - } -} - - -sink { - Redis { - host = "redis-e2e" - port = 6379 - db_num=3 - auth = "U2VhVHVubmVs" - key = "key_multi_list" - data_type = list - } -} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-by-db-num.conf deleted file mode 100644 index c32864baaba..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-by-db-num.conf +++ /dev/null @@ -1,54 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = key - db_num=1 - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "db_test" - data_type = list - db_num=2 - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-expire.conf deleted file mode 100644 index a0c5288c542..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis-expire.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = key - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "key_list" - data_type = list - expire = 30 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis.conf deleted file mode 100644 index 1fbbd3cea18..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/redis-to-redis.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = string - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "key_list" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-hash-to-redis-list-hash-check.conf deleted file mode 100644 index 590ec4db0d4..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-hash-to-redis-list-hash-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-hash*" - data_type = hash - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-hash-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf deleted file mode 100644 index 6c03ae9764c..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "list-test-read*" - data_type = list - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-test-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-set-to-redis-list-set-check.conf deleted file mode 100644 index e4c0cfd7616..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-set-to-redis-list-set-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-set*" - data_type = set - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-set-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-string-to-redis.conf deleted file mode 100644 index fabbdb8d732..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-string-to-redis.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "string_test*" - data_type = string - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "string_test_list" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-zset-to-redis-list-zset-check.conf deleted file mode 100644 index 6c24823aabf..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/resources/scan-zset-to-redis-list-zset-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-zset*" - data_type = zset - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-zset-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml deleted file mode 100644 index be93d314104..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/pom.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - 4.0.0 - - org.apache.seatunnel - connector-redis-e2e - ${revision} - - - connector-redis-e2e-redis4 - SeaTunnel : E2E : Connector V2 : Redis : Version4 - - - 8 - 8 - UTF-8 - - - - org.apache.seatunnel - connector-redis-e2e-common - ${revision} - test-jar - test - - - - diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/fake-to-multipletableredissink.conf deleted file mode 100644 index 051c51eea66..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/fake-to-multipletableredissink.conf +++ /dev/null @@ -1,97 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -###### -###### This config file is a demonstration of streaming processing in seatunnel config -###### -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} -source { - FakeSource { - tables_configs = [ - { - schema = { - table = "redis_sink_1" - fields { - id = int - val_bool = boolean - val_int8 = tinyint - val_int16 = smallint - val_int32 = int - val_int64 = bigint - val_float = float - val_double = double - val_decimal = "decimal(16, 1)" - val_string = string - val_unixtime_micros = timestamp - } - } - rows = [ - { - kind = INSERT - fields = [1, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] - } - ] - }, - { - schema = { - table = "redis_sink_2" - fields { - id = int - val_bool = boolean - val_int8 = tinyint - val_int16 = smallint - val_int32 = int - val_int64 = bigint - val_float = float - val_double = double - val_decimal = "decimal(16, 1)" - val_string = string - val_unixtime_micros = timestamp - } - } - rows = [ - { - kind = INSERT - fields = [2, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] - } - ] - } - ] - } -} - - -sink { - Redis { - host = "redis-e2e" - port = 6379 - db_num=3 - auth = "U2VhVHVubmVs" - key = "key_multi_list" - data_type = list - } -} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-by-db-num.conf deleted file mode 100644 index c32864baaba..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-by-db-num.conf +++ /dev/null @@ -1,54 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = key - db_num=1 - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "db_test" - data_type = list - db_num=2 - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-expire.conf deleted file mode 100644 index a0c5288c542..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis-expire.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = key - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "key_list" - data_type = list - expire = 30 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis.conf deleted file mode 100644 index 1fbbd3cea18..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/redis-to-redis.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = string - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "key_list" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-hash-to-redis-list-hash-check.conf deleted file mode 100644 index 590ec4db0d4..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-hash-to-redis-list-hash-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-hash*" - data_type = hash - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-hash-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf deleted file mode 100644 index 6c03ae9764c..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "list-test-read*" - data_type = list - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-test-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-set-to-redis-list-set-check.conf deleted file mode 100644 index e4c0cfd7616..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-set-to-redis-list-set-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-set*" - data_type = set - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-set-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-string-to-redis.conf deleted file mode 100644 index fabbdb8d732..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-string-to-redis.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "string_test*" - data_type = string - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "string_test_list" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-zset-to-redis-list-zset-check.conf deleted file mode 100644 index 6c24823aabf..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/resources/scan-zset-to-redis-list-zset-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-zset*" - data_type = zset - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-zset-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml deleted file mode 100644 index b93195e5557..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - - org.apache.seatunnel - connector-redis-e2e - ${revision} - - - connector-redis-e2e-redis5 - - SeaTunnel : E2E : Connector V2 : Redis : Version5 - - - 8 - 8 - UTF-8 - - - - org.apache.seatunnel - connector-redis-e2e-common - ${revision} - test-jar - test - - - - diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/fake-to-multipletableredissink.conf deleted file mode 100644 index 051c51eea66..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/fake-to-multipletableredissink.conf +++ /dev/null @@ -1,97 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -###### -###### This config file is a demonstration of streaming processing in seatunnel config -###### -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} -source { - FakeSource { - tables_configs = [ - { - schema = { - table = "redis_sink_1" - fields { - id = int - val_bool = boolean - val_int8 = tinyint - val_int16 = smallint - val_int32 = int - val_int64 = bigint - val_float = float - val_double = double - val_decimal = "decimal(16, 1)" - val_string = string - val_unixtime_micros = timestamp - } - } - rows = [ - { - kind = INSERT - fields = [1, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] - } - ] - }, - { - schema = { - table = "redis_sink_2" - fields { - id = int - val_bool = boolean - val_int8 = tinyint - val_int16 = smallint - val_int32 = int - val_int64 = bigint - val_float = float - val_double = double - val_decimal = "decimal(16, 1)" - val_string = string - val_unixtime_micros = timestamp - } - } - rows = [ - { - kind = INSERT - fields = [2, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] - } - ] - } - ] - } -} - - -sink { - Redis { - host = "redis-e2e" - port = 6379 - db_num=3 - auth = "U2VhVHVubmVs" - key = "key_multi_list" - data_type = list - } -} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-by-db-num.conf deleted file mode 100644 index c32864baaba..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-by-db-num.conf +++ /dev/null @@ -1,54 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = key - db_num=1 - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "db_test" - data_type = list - db_num=2 - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-expire.conf deleted file mode 100644 index a0c5288c542..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis-expire.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = key - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "key_list" - data_type = list - expire = 30 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis.conf deleted file mode 100644 index 1fbbd3cea18..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/redis-to-redis.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = string - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "key_list" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-hash-to-redis-list-hash-check.conf deleted file mode 100644 index 590ec4db0d4..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-hash-to-redis-list-hash-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-hash*" - data_type = hash - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-hash-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf deleted file mode 100644 index 6c03ae9764c..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "list-test-read*" - data_type = list - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-test-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-set-to-redis-list-set-check.conf deleted file mode 100644 index e4c0cfd7616..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-set-to-redis-list-set-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-set*" - data_type = set - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-set-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-string-to-redis.conf deleted file mode 100644 index fabbdb8d732..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-string-to-redis.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "string_test*" - data_type = string - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "string_test_list" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-zset-to-redis-list-zset-check.conf deleted file mode 100644 index 6c24823aabf..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/resources/scan-zset-to-redis-list-zset-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-zset*" - data_type = zset - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-zset-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml deleted file mode 100644 index 45e4e0ce18b..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - - org.apache.seatunnel - connector-redis-e2e - ${revision} - - - connector-redis-e2e-redis6 - - SeaTunnel : E2E : Connector V2 : Redis : Version6 - - - 8 - 8 - UTF-8 - - - - org.apache.seatunnel - connector-redis-e2e-common - ${revision} - test-jar - test - - - - diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/fake-to-multipletableredissink.conf deleted file mode 100644 index 051c51eea66..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/fake-to-multipletableredissink.conf +++ /dev/null @@ -1,97 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -###### -###### This config file is a demonstration of streaming processing in seatunnel config -###### -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} -source { - FakeSource { - tables_configs = [ - { - schema = { - table = "redis_sink_1" - fields { - id = int - val_bool = boolean - val_int8 = tinyint - val_int16 = smallint - val_int32 = int - val_int64 = bigint - val_float = float - val_double = double - val_decimal = "decimal(16, 1)" - val_string = string - val_unixtime_micros = timestamp - } - } - rows = [ - { - kind = INSERT - fields = [1, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] - } - ] - }, - { - schema = { - table = "redis_sink_2" - fields { - id = int - val_bool = boolean - val_int8 = tinyint - val_int16 = smallint - val_int32 = int - val_int64 = bigint - val_float = float - val_double = double - val_decimal = "decimal(16, 1)" - val_string = string - val_unixtime_micros = timestamp - } - } - rows = [ - { - kind = INSERT - fields = [2, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] - } - ] - } - ] - } -} - - -sink { - Redis { - host = "redis-e2e" - port = 6379 - db_num=3 - auth = "U2VhVHVubmVs" - key = "key_multi_list" - data_type = list - } -} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-by-db-num.conf deleted file mode 100644 index c32864baaba..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-by-db-num.conf +++ /dev/null @@ -1,54 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = key - db_num=1 - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "db_test" - data_type = list - db_num=2 - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-expire.conf deleted file mode 100644 index a0c5288c542..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis-expire.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = key - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "key_list" - data_type = list - expire = 30 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis.conf deleted file mode 100644 index 1fbbd3cea18..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/redis-to-redis.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = string - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "key_list" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-hash-to-redis-list-hash-check.conf deleted file mode 100644 index 590ec4db0d4..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-hash-to-redis-list-hash-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-hash*" - data_type = hash - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-hash-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf deleted file mode 100644 index 6c03ae9764c..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "list-test-read*" - data_type = list - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-test-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-set-to-redis-list-set-check.conf deleted file mode 100644 index e4c0cfd7616..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-set-to-redis-list-set-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-set*" - data_type = set - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-set-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-string-to-redis.conf deleted file mode 100644 index fabbdb8d732..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-string-to-redis.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "string_test*" - data_type = string - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "string_test_list" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-zset-to-redis-list-zset-check.conf deleted file mode 100644 index 6c24823aabf..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/resources/scan-zset-to-redis-list-zset-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-zset*" - data_type = zset - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-zset-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml deleted file mode 100644 index d74a4855820..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - - org.apache.seatunnel - connector-redis-e2e - ${revision} - - - connector-redis-e2e-redis7 - - SeaTunnel : E2E : Connector V2 : Redis : Version7 - - - 8 - 8 - UTF-8 - - - - org.apache.seatunnel - connector-redis-e2e-common - ${revision} - test-jar - test - - - - diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/fake-to-multipletableredissink.conf deleted file mode 100644 index 051c51eea66..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/fake-to-multipletableredissink.conf +++ /dev/null @@ -1,97 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -###### -###### This config file is a demonstration of streaming processing in seatunnel config -###### -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} -source { - FakeSource { - tables_configs = [ - { - schema = { - table = "redis_sink_1" - fields { - id = int - val_bool = boolean - val_int8 = tinyint - val_int16 = smallint - val_int32 = int - val_int64 = bigint - val_float = float - val_double = double - val_decimal = "decimal(16, 1)" - val_string = string - val_unixtime_micros = timestamp - } - } - rows = [ - { - kind = INSERT - fields = [1, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] - } - ] - }, - { - schema = { - table = "redis_sink_2" - fields { - id = int - val_bool = boolean - val_int8 = tinyint - val_int16 = smallint - val_int32 = int - val_int64 = bigint - val_float = float - val_double = double - val_decimal = "decimal(16, 1)" - val_string = string - val_unixtime_micros = timestamp - } - } - rows = [ - { - kind = INSERT - fields = [2, true, 1, 2, 3, 4, 4.3,5.3,6.3, "lzl", "2020-02-02T02:02:02"] - } - ] - } - ] - } -} - - -sink { - Redis { - host = "redis-e2e" - port = 6379 - db_num=3 - auth = "U2VhVHVubmVs" - key = "key_multi_list" - data_type = list - } -} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-by-db-num.conf deleted file mode 100644 index c32864baaba..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-by-db-num.conf +++ /dev/null @@ -1,54 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = key - db_num=1 - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "db_test" - data_type = list - db_num=2 - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-expire.conf deleted file mode 100644 index a0c5288c542..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis-expire.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = key - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "key_list" - data_type = list - expire = 30 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis.conf deleted file mode 100644 index 1fbbd3cea18..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/redis-to-redis.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = string - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "key_list" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-hash-to-redis-list-hash-check.conf deleted file mode 100644 index 590ec4db0d4..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-hash-to-redis-list-hash-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-hash*" - data_type = hash - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-hash-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf deleted file mode 100644 index 6c03ae9764c..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "list-test-read*" - data_type = list - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-test-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-set-to-redis-list-set-check.conf deleted file mode 100644 index e4c0cfd7616..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-set-to-redis-list-set-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-set*" - data_type = set - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-set-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-string-to-redis.conf deleted file mode 100644 index fabbdb8d732..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-string-to-redis.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "string_test*" - data_type = string - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "string_test_list" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-zset-to-redis-list-zset-check.conf deleted file mode 100644 index 6c24823aabf..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/resources/scan-zset-to-redis-list-zset-check.conf +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - -source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key-test-zset*" - data_type = zset - batch_size = 33 - redis_version = ${redisVersion} - } -} - -sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "list-zset-check" - data_type = list - batch_size = 33 - } -} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/pom.xml index 042e6eb889c..5f3283e805a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/pom.xml @@ -23,28 +23,8 @@ connector-redis-e2e - pom SeaTunnel : E2E : Connector V2 : Redis - - connector-redis-e2e-common - connector-redis-e2e-redis3 - connector-redis-e2e-redis4 - connector-redis-e2e-redis5 - connector-redis-e2e-redis6 - connector-redis-e2e-redis7 - - - - - org.apache.seatunnel - connector-redis - ${project.version} - pom - import - - - diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java similarity index 81% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java index 96cd71cd262..9fce4bf62d9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis3/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java @@ -19,15 +19,15 @@ import java.util.Collections; import java.util.List; -public class Redis3IT extends RedisTestCaseTemplateIT { +public class Redis3IT extends RedisIT { @Override - public RedisContainerInfo getRedisContainerInfo() { - return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:3"); + public List getVariable() { + return Collections.singletonList("redisVersion=Redis3"); } @Override - public List getVariables() { - return Collections.singletonList("redisVersion=Redis3"); + public String getImage() { + return "redis:3"; } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java similarity index 81% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java index dd6bd3ee6c2..42b3a159439 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis4/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java @@ -19,15 +19,15 @@ import java.util.Collections; import java.util.List; -public class Redis4IT extends RedisTestCaseTemplateIT { +public class Redis4IT extends RedisIT { @Override - public RedisContainerInfo getRedisContainerInfo() { - return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:4"); + public List getVariable() { + return Collections.singletonList("redisVersion=Redis4"); } @Override - public List getVariables() { - return Collections.singletonList("redisVersion=Redis4"); + public String getImage() { + return "redis:4"; } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java similarity index 81% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java index 347dedb4cfe..351ab60bd92 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis5/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java @@ -19,15 +19,14 @@ import java.util.Collections; import java.util.List; -public class Redis5IT extends RedisTestCaseTemplateIT { - +public class Redis5IT extends RedisIT { @Override - public RedisContainerInfo getRedisContainerInfo() { - return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:5"); + public List getVariable() { + return Collections.singletonList("redisVersion=Redis5"); } @Override - public List getVariables() { - return Collections.singletonList("redisVersion=Redis5"); + public String getImage() { + return "redis:5"; } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java similarity index 81% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java index 6c5a0295f99..e24f8d1307e 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis6/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java @@ -19,15 +19,15 @@ import java.util.Collections; import java.util.List; -public class Redis6IT extends RedisTestCaseTemplateIT { +public class Redis6IT extends RedisIT { @Override - public RedisContainerInfo getRedisContainerInfo() { - return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:6"); + public List getVariable() { + return Collections.singletonList("redisVersion=Redis6"); } @Override - public List getVariables() { - return Collections.singletonList("redisVersion=Redis6"); + public String getImage() { + return "redis:6"; } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java similarity index 81% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java index 45616620c72..a89d80de2c5 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-redis7/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java @@ -19,15 +19,15 @@ import java.util.Collections; import java.util.List; -public class Redis7IT extends RedisTestCaseTemplateIT { +public class Redis7IT extends RedisIT { @Override - public RedisContainerInfo getRedisContainerInfo() { - return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:7"); + public List getVariable() { + return Collections.singletonList("redisVersion=Redis7"); } @Override - public List getVariables() { - return Collections.singletonList("redisVersion=Redis7"); + public String getImage() { + return "redis:7"; } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java similarity index 82% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java index f3964f966d6..bb93ed09604 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.seatunnel.e2e.connector.redis; import org.apache.seatunnel.api.table.type.ArrayType; @@ -62,15 +63,13 @@ import java.util.stream.Stream; @Slf4j -public abstract class RedisTestCaseTemplateIT extends TestSuiteBase implements TestResource { - - private String host; - private int port; - private String password; +public class RedisIT extends TestSuiteBase implements TestResource { - private String imageName; + private final String host = "redis-e2e"; + private final int port = 6379; + private final String password = "SeaTunnel"; - private Pair> testDateSet; + private final Pair> testDateSet = generateTestDataSet(); private GenericContainer redisContainer; @@ -79,14 +78,13 @@ public abstract class RedisTestCaseTemplateIT extends TestSuiteBase implements T @BeforeAll @Override public void startUp() { - initContainerInfo(); this.redisContainer = - new GenericContainer<>(DockerImageName.parse(imageName)) + new GenericContainer<>(DockerImageName.parse(getImage())) .withNetwork(NETWORK) .withNetworkAliases(host) .withExposedPorts(port) .withLogConsumer( - new Slf4jLogConsumer(DockerLoggerFactory.getLogger(imageName))) + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(getImage()))) .withCommand(String.format("redis-server --requirepass %s", password)) .waitingFor( new HostPortWaitStrategy() @@ -97,15 +95,6 @@ public void startUp() { this.initSourceData(); } - private void initContainerInfo() { - RedisContainerInfo redisContainerInfo = getRedisContainerInfo(); - this.host = redisContainerInfo.getHost(); - this.port = redisContainerInfo.getPort(); - this.password = redisContainerInfo.getPassword(); - this.imageName = redisContainerInfo.getImageName(); - this.testDateSet = generateTestDataSet(); - } - private void initSourceData() { JsonSerializationSchema jsonSerializationSchema = new JsonSerializationSchema(testDateSet.getKey()); @@ -122,7 +111,7 @@ private void initSourceData() { jedis.select(0); } - protected Pair> generateTestDataSet() { + private static Pair> generateTestDataSet() { SeaTunnelRowType rowType = new SeaTunnelRowType( new String[] { @@ -207,7 +196,8 @@ public void tearDown() { @TestTemplate public void testRedis(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = container.executeJob(testRedisConf(), getVariables()); + Container.ExecResult execResult = + container.executeJob("/redis-to-redis.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -219,7 +209,7 @@ public void testRedis(TestContainer container) throws IOException, InterruptedEx public void testRedisWithExpire(TestContainer container) throws IOException, InterruptedException { Container.ExecResult execResult = - container.executeJob(testRedisWithExpireConf(), getVariables()); + container.executeJob("/redis-to-redis-expire.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -230,7 +220,7 @@ public void testRedisWithExpire(TestContainer container) @TestTemplate public void testRedisDbNum(TestContainer container) throws IOException, InterruptedException { Container.ExecResult execResult = - container.executeJob(testRedisDbNumConf(), getVariables()); + container.executeJob("/redis-to-redis-by-db-num.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(2); Assertions.assertEquals(100, jedis.llen("db_test")); @@ -246,7 +236,7 @@ public void testScanStringTypeWriteRedis(TestContainer container) jedis.set(keyPrefix + i, "val"); } Container.ExecResult execResult = - container.executeJob(testScanStringTypeWriteRedisConf(), getVariables()); + container.executeJob("/scan-string-to-redis.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("string_test_list", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -267,7 +257,8 @@ public void testScanListTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob(testScanListTypeWriteRedisConf(), getVariables()); + container.executeJob( + "/scan-list-test-read-to-redis-list-test-check.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-test-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -289,7 +280,7 @@ public void testScanSetTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob(testScanSetTypeWriteRedisConf(), getVariables()); + container.executeJob("/scan-set-to-redis-list-set-check.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-set-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -311,7 +302,7 @@ public void testScanHashTypeWriteRedis(TestContainer container) jedis.hset(setKey, map); } Container.ExecResult execResult = - container.executeJob(testScanHashTypeWriteRedisConf(), getVariables()); + container.executeJob("/scan-hash-to-redis-list-hash-check.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-hash-check", 0, -1); Assertions.assertEquals(100, list.size()); @@ -339,7 +330,7 @@ public void testScanZsetTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob(testScanZsetTypeWriteRedisConf(), getVariables()); + container.executeJob("/scan-zset-to-redis-list-zset-check.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-zset-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -358,7 +349,7 @@ public void testScanZsetTypeWriteRedis(TestContainer container) public void testMultipletableRedisSink(TestContainer container) throws IOException, InterruptedException { Container.ExecResult execResult = - container.executeJob(testMultipletableRedisSinkConf(), getVariables()); + container.executeJob("/fake-to-multipletableredissink.conf", getVariable()); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(3); Assertions.assertEquals(2, jedis.llen("key_multi_list")); @@ -366,45 +357,11 @@ public void testMultipletableRedisSink(TestContainer container) jedis.select(0); } - ///// need different redis version test override - - public String testRedisConf() { - return "/redis-to-redis.conf"; - } - - public String testRedisWithExpireConf() { - return "/redis-to-redis-expire.conf"; - } - - public String testRedisDbNumConf() { - return "/redis-to-redis-by-db-num.conf"; - } - - public String testScanStringTypeWriteRedisConf() { - return "/scan-string-to-redis.conf"; - } - - public String testScanListTypeWriteRedisConf() { - return "/scan-list-test-read-to-redis-list-test-check.conf"; + public List getVariable() { + return Collections.singletonList("redisVersion=Redis7"); } - public String testScanHashTypeWriteRedisConf() { - return "/scan-hash-to-redis-list-hash-check.conf"; + public String getImage() { + return "redis:latest"; } - - public String testScanZsetTypeWriteRedisConf() { - return "/scan-zset-to-redis-list-zset-check.conf"; - } - - public String testScanSetTypeWriteRedisConf() { - return "/scan-set-to-redis-list-set-check.conf"; - } - - public String testMultipletableRedisSinkConf() { - return "/fake-to-multipletableredissink.conf"; - } - - public abstract RedisContainerInfo getRedisContainerInfo(); - - public abstract List getVariables(); } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/fake-to-multipletableredissink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/fake-to-multipletableredissink.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/fake-to-multipletableredissink.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/fake-to-multipletableredissink.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis-by-db-num.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis-expire.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/redis-to-redis.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-hash-to-redis-list-hash-check.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-set-to-redis-list-set-check.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-string-to-redis.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf similarity index 100% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/connector-redis-e2e-common/src/test/resources/scan-zset-to-redis-list-zset-check.conf rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf From d96edb8cfdd3085aacd482113e89d3876ca0aa8d Mon Sep 17 00:00:00 2001 From: FuYouJ <1247908487@qq.com> Date: Wed, 18 Sep 2024 23:49:26 +0800 Subject: [PATCH 07/10] [improve][Redis]Merge redis 5,7 tests into the same module --- .../e2e/connector/redis/Redis4IT.java | 33 ------- .../e2e/connector/redis/Redis5IT.java | 11 ++- .../e2e/connector/redis/Redis6IT.java | 33 ------- .../e2e/connector/redis/Redis7IT.java | 10 +- ...{Redis3IT.java => RedisContainerInfo.java} | 32 +++++-- ...isIT.java => RedisTestCaseTemplateIT.java} | 91 ++++++++++++++----- 6 files changed, 101 insertions(+), 109 deletions(-) delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/{Redis3IT.java => RedisContainerInfo.java} (58%) rename seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/{RedisIT.java => RedisTestCaseTemplateIT.java} (82%) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java deleted file mode 100644 index 42b3a159439..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis4IT.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.seatunnel.e2e.connector.redis; - -import java.util.Collections; -import java.util.List; - -public class Redis4IT extends RedisIT { - - @Override - public List getVariable() { - return Collections.singletonList("redisVersion=Redis4"); - } - - @Override - public String getImage() { - return "redis:4"; - } -} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java index 351ab60bd92..347dedb4cfe 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java @@ -19,14 +19,15 @@ import java.util.Collections; import java.util.List; -public class Redis5IT extends RedisIT { +public class Redis5IT extends RedisTestCaseTemplateIT { + @Override - public List getVariable() { - return Collections.singletonList("redisVersion=Redis5"); + public RedisContainerInfo getRedisContainerInfo() { + return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:5"); } @Override - public String getImage() { - return "redis:5"; + public List getVariables() { + return Collections.singletonList("redisVersion=Redis5"); } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java deleted file mode 100644 index e24f8d1307e..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis6IT.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.seatunnel.e2e.connector.redis; - -import java.util.Collections; -import java.util.List; - -public class Redis6IT extends RedisIT { - - @Override - public List getVariable() { - return Collections.singletonList("redisVersion=Redis6"); - } - - @Override - public String getImage() { - return "redis:6"; - } -} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java index a89d80de2c5..45616620c72 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java @@ -19,15 +19,15 @@ import java.util.Collections; import java.util.List; -public class Redis7IT extends RedisIT { +public class Redis7IT extends RedisTestCaseTemplateIT { @Override - public List getVariable() { - return Collections.singletonList("redisVersion=Redis7"); + public RedisContainerInfo getRedisContainerInfo() { + return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:7"); } @Override - public String getImage() { - return "redis:7"; + public List getVariables() { + return Collections.singletonList("redisVersion=Redis7"); } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisContainerInfo.java similarity index 58% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisContainerInfo.java index 9fce4bf62d9..61b55a65944 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis3IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisContainerInfo.java @@ -16,18 +16,32 @@ */ package org.apache.seatunnel.e2e.connector.redis; -import java.util.Collections; -import java.util.List; +public class RedisContainerInfo { + private final String host; + private final int port; + private final String password; + private final String imageName; -public class Redis3IT extends RedisIT { + public RedisContainerInfo(String host, int port, String password, String imageName) { + this.host = host; + this.port = port; + this.password = password; + this.imageName = imageName; + } + + public String getHost() { + return host; + } + + public int getPort() { + return port; + } - @Override - public List getVariable() { - return Collections.singletonList("redisVersion=Redis3"); + public String getPassword() { + return password; } - @Override - public String getImage() { - return "redis:3"; + public String getImageName() { + return imageName; } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java similarity index 82% rename from seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java rename to seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java index bb93ed09604..f3964f966d6 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.seatunnel.e2e.connector.redis; import org.apache.seatunnel.api.table.type.ArrayType; @@ -63,13 +62,15 @@ import java.util.stream.Stream; @Slf4j -public class RedisIT extends TestSuiteBase implements TestResource { +public abstract class RedisTestCaseTemplateIT extends TestSuiteBase implements TestResource { + + private String host; + private int port; + private String password; - private final String host = "redis-e2e"; - private final int port = 6379; - private final String password = "SeaTunnel"; + private String imageName; - private final Pair> testDateSet = generateTestDataSet(); + private Pair> testDateSet; private GenericContainer redisContainer; @@ -78,13 +79,14 @@ public class RedisIT extends TestSuiteBase implements TestResource { @BeforeAll @Override public void startUp() { + initContainerInfo(); this.redisContainer = - new GenericContainer<>(DockerImageName.parse(getImage())) + new GenericContainer<>(DockerImageName.parse(imageName)) .withNetwork(NETWORK) .withNetworkAliases(host) .withExposedPorts(port) .withLogConsumer( - new Slf4jLogConsumer(DockerLoggerFactory.getLogger(getImage()))) + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(imageName))) .withCommand(String.format("redis-server --requirepass %s", password)) .waitingFor( new HostPortWaitStrategy() @@ -95,6 +97,15 @@ public void startUp() { this.initSourceData(); } + private void initContainerInfo() { + RedisContainerInfo redisContainerInfo = getRedisContainerInfo(); + this.host = redisContainerInfo.getHost(); + this.port = redisContainerInfo.getPort(); + this.password = redisContainerInfo.getPassword(); + this.imageName = redisContainerInfo.getImageName(); + this.testDateSet = generateTestDataSet(); + } + private void initSourceData() { JsonSerializationSchema jsonSerializationSchema = new JsonSerializationSchema(testDateSet.getKey()); @@ -111,7 +122,7 @@ private void initSourceData() { jedis.select(0); } - private static Pair> generateTestDataSet() { + protected Pair> generateTestDataSet() { SeaTunnelRowType rowType = new SeaTunnelRowType( new String[] { @@ -196,8 +207,7 @@ public void tearDown() { @TestTemplate public void testRedis(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = - container.executeJob("/redis-to-redis.conf", getVariable()); + Container.ExecResult execResult = container.executeJob(testRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -209,7 +219,7 @@ public void testRedis(TestContainer container) throws IOException, InterruptedEx public void testRedisWithExpire(TestContainer container) throws IOException, InterruptedException { Container.ExecResult execResult = - container.executeJob("/redis-to-redis-expire.conf", getVariable()); + container.executeJob(testRedisWithExpireConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -220,7 +230,7 @@ public void testRedisWithExpire(TestContainer container) @TestTemplate public void testRedisDbNum(TestContainer container) throws IOException, InterruptedException { Container.ExecResult execResult = - container.executeJob("/redis-to-redis-by-db-num.conf", getVariable()); + container.executeJob(testRedisDbNumConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(2); Assertions.assertEquals(100, jedis.llen("db_test")); @@ -236,7 +246,7 @@ public void testScanStringTypeWriteRedis(TestContainer container) jedis.set(keyPrefix + i, "val"); } Container.ExecResult execResult = - container.executeJob("/scan-string-to-redis.conf", getVariable()); + container.executeJob(testScanStringTypeWriteRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("string_test_list", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -257,8 +267,7 @@ public void testScanListTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob( - "/scan-list-test-read-to-redis-list-test-check.conf", getVariable()); + container.executeJob(testScanListTypeWriteRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-test-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -280,7 +289,7 @@ public void testScanSetTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob("/scan-set-to-redis-list-set-check.conf", getVariable()); + container.executeJob(testScanSetTypeWriteRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-set-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -302,7 +311,7 @@ public void testScanHashTypeWriteRedis(TestContainer container) jedis.hset(setKey, map); } Container.ExecResult execResult = - container.executeJob("/scan-hash-to-redis-list-hash-check.conf", getVariable()); + container.executeJob(testScanHashTypeWriteRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-hash-check", 0, -1); Assertions.assertEquals(100, list.size()); @@ -330,7 +339,7 @@ public void testScanZsetTypeWriteRedis(TestContainer container) } } Container.ExecResult execResult = - container.executeJob("/scan-zset-to-redis-list-zset-check.conf", getVariable()); + container.executeJob(testScanZsetTypeWriteRedisConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-zset-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -349,7 +358,7 @@ public void testScanZsetTypeWriteRedis(TestContainer container) public void testMultipletableRedisSink(TestContainer container) throws IOException, InterruptedException { Container.ExecResult execResult = - container.executeJob("/fake-to-multipletableredissink.conf", getVariable()); + container.executeJob(testMultipletableRedisSinkConf(), getVariables()); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(3); Assertions.assertEquals(2, jedis.llen("key_multi_list")); @@ -357,11 +366,45 @@ public void testMultipletableRedisSink(TestContainer container) jedis.select(0); } - public List getVariable() { - return Collections.singletonList("redisVersion=Redis7"); + ///// need different redis version test override + + public String testRedisConf() { + return "/redis-to-redis.conf"; + } + + public String testRedisWithExpireConf() { + return "/redis-to-redis-expire.conf"; + } + + public String testRedisDbNumConf() { + return "/redis-to-redis-by-db-num.conf"; + } + + public String testScanStringTypeWriteRedisConf() { + return "/scan-string-to-redis.conf"; + } + + public String testScanListTypeWriteRedisConf() { + return "/scan-list-test-read-to-redis-list-test-check.conf"; } - public String getImage() { - return "redis:latest"; + public String testScanHashTypeWriteRedisConf() { + return "/scan-hash-to-redis-list-hash-check.conf"; } + + public String testScanZsetTypeWriteRedisConf() { + return "/scan-zset-to-redis-list-zset-check.conf"; + } + + public String testScanSetTypeWriteRedisConf() { + return "/scan-set-to-redis-list-set-check.conf"; + } + + public String testMultipletableRedisSinkConf() { + return "/fake-to-multipletableredissink.conf"; + } + + public abstract RedisContainerInfo getRedisContainerInfo(); + + public abstract List getVariables(); } From d94b171b95c56a9ad1b387878b0bca102c14d8e1 Mon Sep 17 00:00:00 2001 From: FuYouJ <1247908487@qq.com> Date: Fri, 20 Sep 2024 00:14:32 +0800 Subject: [PATCH 08/10] =?UTF-8?q?[improve][Redis]remove=20redis=5Fversion?= =?UTF-8?q?=20config=20option=EF=BC=8Cuse=20jedis.info()=20to=20get=20redi?= =?UTF-8?q?s=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/en/connector-v2/source/Redis.md | 5 -- .../seatunnel/redis/client/RedisClient.java | 15 +++--- .../redis/client/RedisClusterClient.java | 4 +- .../redis/client/RedisSingleClient.java | 4 +- .../seatunnel/redis/config/RedisConfig.java | 6 --- .../redis/config/RedisParameters.java | 51 +++++++++++++++---- .../RedisErrorCode.java} | 31 ++++++++--- .../redis/source/RedisSourceFactory.java | 3 +- .../e2e/connector/redis/Redis5IT.java | 8 --- .../e2e/connector/redis/Redis7IT.java | 8 --- .../redis/RedisTestCaseTemplateIT.java | 28 ++++------ .../resources/redis-to-redis-by-db-num.conf | 1 - .../test/resources/redis-to-redis-expire.conf | 1 - .../src/test/resources/redis-to-redis.conf | 1 - .../scan-hash-to-redis-list-hash-check.conf | 1 - ...st-test-read-to-redis-list-test-check.conf | 1 - .../scan-set-to-redis-list-set-check.conf | 1 - .../test/resources/scan-string-to-redis.conf | 1 - .../scan-zset-to-redis-list-zset-check.conf | 1 - 19 files changed, 85 insertions(+), 86 deletions(-) rename seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/{config/RedisVersion.java => exception/RedisErrorCode.java} (53%) diff --git a/docs/en/connector-v2/source/Redis.md b/docs/en/connector-v2/source/Redis.md index f7eedb0618f..dcaa24eb4df 100644 --- a/docs/en/connector-v2/source/Redis.md +++ b/docs/en/connector-v2/source/Redis.md @@ -19,7 +19,6 @@ Used to read data from Redis. | name | type | required | default value | | ------------------- | ------ | --------------------- | ------------- | -| redis_version | string | yes | - | | host | string | yes | - | | port | int | yes | - | | keys | string | yes | - | @@ -35,10 +34,6 @@ Used to read data from Redis. | format | string | no | json | | common-options | | no | - | -### redis_version - -version of redis, support `Redis3`,` Redis4`, `Redis5`,` Redis6`,` Redis7` - ### host [string] redis host diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClient.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClient.java index a96a221ff31..109c51f78a3 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClient.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClient.java @@ -19,7 +19,6 @@ import org.apache.seatunnel.connectors.seatunnel.redis.config.RedisDataType; import org.apache.seatunnel.connectors.seatunnel.redis.config.RedisParameters; -import org.apache.seatunnel.connectors.seatunnel.redis.config.RedisVersion; import redis.clients.jedis.Jedis; import redis.clients.jedis.params.ScanParams; @@ -34,17 +33,19 @@ public abstract class RedisClient extends Jedis { protected final RedisParameters redisParameters; - private final RedisVersion redisVersion; + private final Integer redisVersion; protected final int batchSize; protected final Jedis jedis; - protected RedisClient(RedisParameters redisParameters, Jedis jedis) { + private static final int REDIS_5 = 5; + + protected RedisClient(RedisParameters redisParameters, Jedis jedis, int redisVersion) { this.redisParameters = redisParameters; this.batchSize = redisParameters.getBatchSize(); this.jedis = jedis; - this.redisVersion = redisParameters.getRedisVersion(); + this.redisVersion = redisVersion; } public ScanResult scanKeys( @@ -56,10 +57,8 @@ public ScanResult scanKeys( } private ScanResult scanByRedisVersion( - String cursor, ScanParams scanParams, RedisDataType type, RedisVersion redisVersion) { - if (RedisVersion.Redis3.equals(redisVersion) - || RedisVersion.Redis4.equals(redisVersion) - || RedisVersion.Redis5.equals(redisVersion)) { + String cursor, ScanParams scanParams, RedisDataType type, Integer redisVersion) { + if (redisVersion <= REDIS_5) { return scanOnRedis5(cursor, scanParams, type); } else { return jedis.scan(cursor, scanParams, type.name()); diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClusterClient.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClusterClient.java index 13acc89defe..bd687e6c9b9 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClusterClient.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisClusterClient.java @@ -30,8 +30,8 @@ import java.util.Set; public class RedisClusterClient extends RedisClient { - public RedisClusterClient(RedisParameters redisParameters, Jedis jedis) { - super(redisParameters, jedis); + public RedisClusterClient(RedisParameters redisParameters, Jedis jedis, int redisVersion) { + super(redisParameters, jedis, redisVersion); } @Override diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisSingleClient.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisSingleClient.java index 99bae5e7333..f79aa46e98c 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisSingleClient.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/client/RedisSingleClient.java @@ -34,8 +34,8 @@ // In standalone mode, pipeline can be used to improve batch read performance public class RedisSingleClient extends RedisClient { - public RedisSingleClient(RedisParameters redisParameters, Jedis jedis) { - super(redisParameters, jedis); + public RedisSingleClient(RedisParameters redisParameters, Jedis jedis, int redisVersion) { + super(redisParameters, jedis, redisVersion); } @Override diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisConfig.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisConfig.java index a88c0e5726e..3be5b39de99 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisConfig.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisConfig.java @@ -36,12 +36,6 @@ public enum HashKeyParseMode { KV; } - public static final Option REDIS_VERSION = - Options.key("redis_version") - .enumType(RedisVersion.class) - .noDefaultValue() - .withDescription("the version of redis,minimum supported Redis3"); - public static final Option HOST = Options.key("host") .stringType() diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisParameters.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisParameters.java index 90eaa78ac2c..3d7e954f1d1 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisParameters.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisParameters.java @@ -18,7 +18,7 @@ package org.apache.seatunnel.connectors.seatunnel.redis.config; import org.apache.seatunnel.api.configuration.ReadonlyConfig; -import org.apache.seatunnel.common.exception.CommonErrorCodeDeprecated; +import org.apache.seatunnel.common.exception.CommonErrorCode; import org.apache.seatunnel.connectors.seatunnel.redis.client.RedisClient; import org.apache.seatunnel.connectors.seatunnel.redis.client.RedisClusterClient; import org.apache.seatunnel.connectors.seatunnel.redis.client.RedisSingleClient; @@ -27,6 +27,7 @@ import org.apache.commons.lang3.StringUtils; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import redis.clients.jedis.ConnectionPoolConfig; import redis.clients.jedis.HostAndPort; import redis.clients.jedis.Jedis; @@ -36,9 +37,12 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; + +import static org.apache.seatunnel.connectors.seatunnel.redis.exception.RedisErrorCode.GET_REDIS_VERSION_INFO_FAILED; +import static org.apache.seatunnel.connectors.seatunnel.redis.exception.RedisErrorCode.INVALID_CONFIG; @Data +@Slf4j public class RedisParameters implements Serializable { private String host; private int port; @@ -54,12 +58,9 @@ public class RedisParameters implements Serializable { private long expire = RedisConfig.EXPIRE.defaultValue(); private int batchSize = RedisConfig.BATCH_SIZE.defaultValue(); - private RedisVersion redisVersion; + private int redisVersion; public void buildWithConfig(ReadonlyConfig config) { - // redis version - Optional versionOptional = config.getOptional(RedisConfig.REDIS_VERSION); - versionOptional.ifPresent(version -> this.redisVersion = version); // set host this.host = config.get(RedisConfig.HOST); // set port @@ -100,11 +101,40 @@ public void buildWithConfig(ReadonlyConfig config) { public RedisClient buildRedisClient() { Jedis jedis = this.buildJedis(); + this.redisVersion = extractRedisVersion(jedis); if (mode.equals(RedisConfig.RedisMode.SINGLE)) { - return new RedisSingleClient(this, jedis); + return new RedisSingleClient(this, jedis, redisVersion); } else { - return new RedisClusterClient(this, jedis); + return new RedisClusterClient(this, jedis, redisVersion); + } + } + + private int extractRedisVersion(Jedis jedis) { + log.info("Try to get redis version information from the jedis.info() method"); + // # Server + // redis_version:5.0.14 + // redis_git_sha1:00000000 + // redis_git_dirty:0 + String info = jedis.info(); + try { + for (String line : info.split("\n")) { + if (line.startsWith("redis_version:")) { + // 5.0.14 + String versionInfo = line.split(":")[1].trim(); + log.info("The version of Redis is :{}", versionInfo); + String[] parts = versionInfo.split("\\."); + return Integer.parseInt(parts[0]); + } + } + } catch (Exception e) { + throw new RedisConnectorException( + GET_REDIS_VERSION_INFO_FAILED, + GET_REDIS_VERSION_INFO_FAILED.getErrorMessage(), + e); } + throw new RedisConnectorException( + GET_REDIS_VERSION_INFO_FAILED, + "Did not get the expected redis_version from the jedis.info() method"); } public Jedis buildJedis() { @@ -128,7 +158,7 @@ public Jedis buildJedis() { String[] splits = redisNode.split(":"); if (splits.length != 2) { throw new RedisConnectorException( - CommonErrorCodeDeprecated.ILLEGAL_ARGUMENT, + INVALID_CONFIG, "Invalid redis node information," + "redis node information must like as the following: [host:port]"); } @@ -157,8 +187,7 @@ public Jedis buildJedis() { default: // do nothing throw new RedisConnectorException( - CommonErrorCodeDeprecated.UNSUPPORTED_OPERATION, - "Not support this redis mode"); + CommonErrorCode.OPERATION_NOT_SUPPORTED, "Not support this redis mode"); } } } diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisVersion.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/exception/RedisErrorCode.java similarity index 53% rename from seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisVersion.java rename to seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/exception/RedisErrorCode.java index 17b2cb0fa16..4d9bb745bd9 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/config/RedisVersion.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/exception/RedisErrorCode.java @@ -1,3 +1,4 @@ +package org.apache.seatunnel.connectors.seatunnel.redis.exception; /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -14,12 +15,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.seatunnel.connectors.seatunnel.redis.config; -public enum RedisVersion { - Redis3, - Redis4, - Redis5, - Redis6, - Redis7; +import org.apache.seatunnel.common.exception.SeaTunnelErrorCode; + +public enum RedisErrorCode implements SeaTunnelErrorCode { + GET_REDIS_VERSION_INFO_FAILED("RedisErrorCode-01", "Failed to get the redis version"), + INVALID_CONFIG("RedisErrorCode-02", "Invalid redis Config"); + + private final String code; + private final String description; + + RedisErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return code; + } + + @Override + public String getDescription() { + return description; + } } diff --git a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java index 4fd0403e5a2..c4f9ac099e9 100644 --- a/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java +++ b/seatunnel-connectors-v2/connector-redis/src/main/java/org/apache/seatunnel/connectors/seatunnel/redis/source/RedisSourceFactory.java @@ -51,8 +51,7 @@ public OptionRule optionRule() { RedisConfig.HOST, RedisConfig.PORT, RedisConfig.KEY_PATTERN, - RedisConfig.DATA_TYPE, - RedisConfig.REDIS_VERSION) + RedisConfig.DATA_TYPE) .optional( RedisConfig.MODE, RedisConfig.HASH_KEY_PARSE_MODE, diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java index 347dedb4cfe..6df6e80d6ee 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis5IT.java @@ -16,18 +16,10 @@ */ package org.apache.seatunnel.e2e.connector.redis; -import java.util.Collections; -import java.util.List; - public class Redis5IT extends RedisTestCaseTemplateIT { @Override public RedisContainerInfo getRedisContainerInfo() { return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:5"); } - - @Override - public List getVariables() { - return Collections.singletonList("redisVersion=Redis5"); - } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java index 45616620c72..dfa46e886ae 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/Redis7IT.java @@ -16,18 +16,10 @@ */ package org.apache.seatunnel.e2e.connector.redis; -import java.util.Collections; -import java.util.List; - public class Redis7IT extends RedisTestCaseTemplateIT { @Override public RedisContainerInfo getRedisContainerInfo() { return new RedisContainerInfo("redis-e2e", 6379, "SeaTunnel", "redis:7"); } - - @Override - public List getVariables() { - return Collections.singletonList("redisVersion=Redis7"); - } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java index f3964f966d6..260eea09322 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java @@ -207,7 +207,7 @@ public void tearDown() { @TestTemplate public void testRedis(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = container.executeJob(testRedisConf(), getVariables()); + Container.ExecResult execResult = container.executeJob(testRedisConf()); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -218,8 +218,7 @@ public void testRedis(TestContainer container) throws IOException, InterruptedEx @TestTemplate public void testRedisWithExpire(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = - container.executeJob(testRedisWithExpireConf(), getVariables()); + Container.ExecResult execResult = container.executeJob(testRedisWithExpireConf()); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -229,8 +228,7 @@ public void testRedisWithExpire(TestContainer container) @TestTemplate public void testRedisDbNum(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = - container.executeJob(testRedisDbNumConf(), getVariables()); + Container.ExecResult execResult = container.executeJob(testRedisDbNumConf()); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(2); Assertions.assertEquals(100, jedis.llen("db_test")); @@ -245,8 +243,7 @@ public void testScanStringTypeWriteRedis(TestContainer container) for (int i = 0; i < 1000; i++) { jedis.set(keyPrefix + i, "val"); } - Container.ExecResult execResult = - container.executeJob(testScanStringTypeWriteRedisConf(), getVariables()); + Container.ExecResult execResult = container.executeJob(testScanStringTypeWriteRedisConf()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("string_test_list", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -266,8 +263,7 @@ public void testScanListTypeWriteRedis(TestContainer container) jedis.lpush(list, "val" + j); } } - Container.ExecResult execResult = - container.executeJob(testScanListTypeWriteRedisConf(), getVariables()); + Container.ExecResult execResult = container.executeJob(testScanListTypeWriteRedisConf()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-test-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -288,8 +284,7 @@ public void testScanSetTypeWriteRedis(TestContainer container) jedis.sadd(setKey, j + ""); } } - Container.ExecResult execResult = - container.executeJob(testScanSetTypeWriteRedisConf(), getVariables()); + Container.ExecResult execResult = container.executeJob(testScanSetTypeWriteRedisConf()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-set-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -310,8 +305,7 @@ public void testScanHashTypeWriteRedis(TestContainer container) map.put("name", "fuyoujie"); jedis.hset(setKey, map); } - Container.ExecResult execResult = - container.executeJob(testScanHashTypeWriteRedisConf(), getVariables()); + Container.ExecResult execResult = container.executeJob(testScanHashTypeWriteRedisConf()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-hash-check", 0, -1); Assertions.assertEquals(100, list.size()); @@ -338,8 +332,7 @@ public void testScanZsetTypeWriteRedis(TestContainer container) jedis.zadd(key, 1, j + ""); } } - Container.ExecResult execResult = - container.executeJob(testScanZsetTypeWriteRedisConf(), getVariables()); + Container.ExecResult execResult = container.executeJob(testScanZsetTypeWriteRedisConf()); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-zset-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -357,8 +350,7 @@ public void testScanZsetTypeWriteRedis(TestContainer container) disabledReason = "Currently FLINK do not support multiple table read") public void testMultipletableRedisSink(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = - container.executeJob(testMultipletableRedisSinkConf(), getVariables()); + Container.ExecResult execResult = container.executeJob(testMultipletableRedisSinkConf()); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(3); Assertions.assertEquals(2, jedis.llen("key_multi_list")); @@ -405,6 +397,4 @@ public String testMultipletableRedisSinkConf() { } public abstract RedisContainerInfo getRedisContainerInfo(); - - public abstract List getVariables(); } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf index c32864baaba..9c1dec0c367 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-by-db-num.conf @@ -37,7 +37,6 @@ source { data_type = key db_num=1 batch_size = 33 - redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf index a0c5288c542..133773d799d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis-expire.conf @@ -36,7 +36,6 @@ source { keys = "key_test*" data_type = key batch_size = 33 - redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf index 1fbbd3cea18..20fee261f00 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf @@ -36,7 +36,6 @@ source { keys = "key_test*" data_type = string batch_size = 33 - redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf index 590ec4db0d4..ee76a0a8391 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-hash-to-redis-list-hash-check.conf @@ -36,7 +36,6 @@ source { keys = "key-test-hash*" data_type = hash batch_size = 33 - redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf index 6c03ae9764c..62db2eaa166 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-list-test-read-to-redis-list-test-check.conf @@ -36,7 +36,6 @@ source { keys = "list-test-read*" data_type = list batch_size = 33 - redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf index e4c0cfd7616..4c64224863f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-set-to-redis-list-set-check.conf @@ -36,7 +36,6 @@ source { keys = "key-test-set*" data_type = set batch_size = 33 - redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf index fabbdb8d732..317e603978e 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-string-to-redis.conf @@ -36,7 +36,6 @@ source { keys = "string_test*" data_type = string batch_size = 33 - redis_version = ${redisVersion} } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf index 6c24823aabf..f7d2c05d37a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/scan-zset-to-redis-list-zset-check.conf @@ -36,7 +36,6 @@ source { keys = "key-test-zset*" data_type = zset batch_size = 33 - redis_version = ${redisVersion} } } From a825e4d82094345cbbb6536dec6fada413df033a Mon Sep 17 00:00:00 2001 From: FuYouJ <1247908487@qq.com> Date: Fri, 20 Sep 2024 00:24:50 +0800 Subject: [PATCH 09/10] [improve][Redis]redis.md remove redis_version config option --- docs/en/connector-v2/source/Redis.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/en/connector-v2/source/Redis.md b/docs/en/connector-v2/source/Redis.md index dcaa24eb4df..cc5205c9891 100644 --- a/docs/en/connector-v2/source/Redis.md +++ b/docs/en/connector-v2/source/Redis.md @@ -237,7 +237,6 @@ Redis { keys = "key_test*" data_type = key format = text - redis_version = Redis5 } ``` @@ -248,7 +247,6 @@ Redis { keys = "key_test*" data_type = key format = json - redis_version = Redis5 schema { fields { name = string @@ -282,7 +280,6 @@ source { keys = "string_test*" data_type = string batch_size = 33 - redis_version = Redis7 } } @@ -307,4 +304,4 @@ sink { ### next version - [Improve] Support redis cluster mode connection and user authentication [3188](https://github.com/apache/seatunnel/pull/3188) -- [Bug] Redis scan command supports versions 3, 4, 5, 6, 7 [7666](https://github.com/apache/seatunnel/pull/7666) \ No newline at end of file +- [Bug] Redis scan command supports versions 5, 6, 7 [7666](https://github.com/apache/seatunnel/pull/7666) \ No newline at end of file From 15269ccfbd7d674ec669b0c739fcbd5d972b0dc4 Mon Sep 17 00:00:00 2001 From: FuYouJ <1247908487@qq.com> Date: Sat, 21 Sep 2024 11:24:51 +0800 Subject: [PATCH 10/10] [improve][Redis]Remove useless code abstractions,doc --- docs/en/connector-v2/source/Redis.md | 13 ---- .../redis/RedisTestCaseTemplateIT.java | 61 +++++-------------- 2 files changed, 14 insertions(+), 60 deletions(-) diff --git a/docs/en/connector-v2/source/Redis.md b/docs/en/connector-v2/source/Redis.md index cc5205c9891..bd60830ba32 100644 --- a/docs/en/connector-v2/source/Redis.md +++ b/docs/en/connector-v2/source/Redis.md @@ -259,19 +259,6 @@ Redis { read string type keys write append to list ```hocon -env { - parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" - - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local -} - source { Redis { host = "redis-e2e" diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java index 260eea09322..66288bbb158 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/java/org/apache/seatunnel/e2e/connector/redis/RedisTestCaseTemplateIT.java @@ -207,7 +207,7 @@ public void tearDown() { @TestTemplate public void testRedis(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = container.executeJob(testRedisConf()); + Container.ExecResult execResult = container.executeJob("/redis-to-redis.conf"); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -218,7 +218,7 @@ public void testRedis(TestContainer container) throws IOException, InterruptedEx @TestTemplate public void testRedisWithExpire(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = container.executeJob(testRedisWithExpireConf()); + Container.ExecResult execResult = container.executeJob("/redis-to-redis-expire.conf"); Assertions.assertEquals(0, execResult.getExitCode()); Assertions.assertEquals(100, jedis.llen("key_list")); // Clear data to prevent data duplication in the next TestContainer @@ -228,7 +228,7 @@ public void testRedisWithExpire(TestContainer container) @TestTemplate public void testRedisDbNum(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = container.executeJob(testRedisDbNumConf()); + Container.ExecResult execResult = container.executeJob("/redis-to-redis-by-db-num.conf"); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(2); Assertions.assertEquals(100, jedis.llen("db_test")); @@ -243,7 +243,7 @@ public void testScanStringTypeWriteRedis(TestContainer container) for (int i = 0; i < 1000; i++) { jedis.set(keyPrefix + i, "val"); } - Container.ExecResult execResult = container.executeJob(testScanStringTypeWriteRedisConf()); + Container.ExecResult execResult = container.executeJob("/scan-string-to-redis.conf"); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("string_test_list", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -263,7 +263,8 @@ public void testScanListTypeWriteRedis(TestContainer container) jedis.lpush(list, "val" + j); } } - Container.ExecResult execResult = container.executeJob(testScanListTypeWriteRedisConf()); + Container.ExecResult execResult = + container.executeJob("/scan-list-test-read-to-redis-list-test-check.conf"); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-test-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -284,7 +285,8 @@ public void testScanSetTypeWriteRedis(TestContainer container) jedis.sadd(setKey, j + ""); } } - Container.ExecResult execResult = container.executeJob(testScanSetTypeWriteRedisConf()); + Container.ExecResult execResult = + container.executeJob("/scan-set-to-redis-list-set-check.conf"); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-set-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -305,7 +307,8 @@ public void testScanHashTypeWriteRedis(TestContainer container) map.put("name", "fuyoujie"); jedis.hset(setKey, map); } - Container.ExecResult execResult = container.executeJob(testScanHashTypeWriteRedisConf()); + Container.ExecResult execResult = + container.executeJob("/scan-hash-to-redis-list-hash-check.conf"); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-hash-check", 0, -1); Assertions.assertEquals(100, list.size()); @@ -332,7 +335,8 @@ public void testScanZsetTypeWriteRedis(TestContainer container) jedis.zadd(key, 1, j + ""); } } - Container.ExecResult execResult = container.executeJob(testScanZsetTypeWriteRedisConf()); + Container.ExecResult execResult = + container.executeJob("/scan-zset-to-redis-list-zset-check.conf"); Assertions.assertEquals(0, execResult.getExitCode()); List list = jedis.lrange("list-zset-check", 0, -1); Assertions.assertEquals(1000, list.size()); @@ -350,7 +354,8 @@ public void testScanZsetTypeWriteRedis(TestContainer container) disabledReason = "Currently FLINK do not support multiple table read") public void testMultipletableRedisSink(TestContainer container) throws IOException, InterruptedException { - Container.ExecResult execResult = container.executeJob(testMultipletableRedisSinkConf()); + Container.ExecResult execResult = + container.executeJob("/fake-to-multipletableredissink.conf"); Assertions.assertEquals(0, execResult.getExitCode()); jedis.select(3); Assertions.assertEquals(2, jedis.llen("key_multi_list")); @@ -358,43 +363,5 @@ public void testMultipletableRedisSink(TestContainer container) jedis.select(0); } - ///// need different redis version test override - - public String testRedisConf() { - return "/redis-to-redis.conf"; - } - - public String testRedisWithExpireConf() { - return "/redis-to-redis-expire.conf"; - } - - public String testRedisDbNumConf() { - return "/redis-to-redis-by-db-num.conf"; - } - - public String testScanStringTypeWriteRedisConf() { - return "/scan-string-to-redis.conf"; - } - - public String testScanListTypeWriteRedisConf() { - return "/scan-list-test-read-to-redis-list-test-check.conf"; - } - - public String testScanHashTypeWriteRedisConf() { - return "/scan-hash-to-redis-list-hash-check.conf"; - } - - public String testScanZsetTypeWriteRedisConf() { - return "/scan-zset-to-redis-list-zset-check.conf"; - } - - public String testScanSetTypeWriteRedisConf() { - return "/scan-set-to-redis-list-set-check.conf"; - } - - public String testMultipletableRedisSinkConf() { - return "/fake-to-multipletableredissink.conf"; - } - public abstract RedisContainerInfo getRedisContainerInfo(); }