diff --git a/.github/workflows/release-manual-docker-seatunnel.yml b/.github/workflows/release-manual-docker-seatunnel.yml index 5cd188aee..7dc377b58 100644 --- a/.github/workflows/release-manual-docker-seatunnel.yml +++ b/.github/workflows/release-manual-docker-seatunnel.yml @@ -22,11 +22,11 @@ on: seatunnelVersion: description: 'seatunnel version' required: true - default: '2.3.7' + default: '2.3.8' type: choice options: + - 2.3.8 - 2.3.7 - - 2.3.6 flinkVersion: description: 'flink version' required: true diff --git a/README.md b/README.md index 65d8e917c..21dc58efd 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Scaleph is driven by personal interest and evolves actively through faithful dev * Data Integration * Web-ui click-and-drag data integration ways backended by [Apache SeaTunnel](https://seatunnel.apache.org/) on Flink engine. - * Support the latest 2.3.7 V2 out-of-the-box connectors and transforms. + * Support the latest 2.3.8 V2 out-of-the-box connectors and transforms. * DataSource management. diff --git a/scaleph-application/scaleph-application-flink/src/main/java/cn/sliew/scaleph/application/flink/resource/handler/FlinkImageMapping.java b/scaleph-application/scaleph-application-flink/src/main/java/cn/sliew/scaleph/application/flink/resource/handler/FlinkImageMapping.java index 7ffbf4744..da280ecc3 100644 --- a/scaleph-application/scaleph-application-flink/src/main/java/cn/sliew/scaleph/application/flink/resource/handler/FlinkImageMapping.java +++ b/scaleph-application/scaleph-application-flink/src/main/java/cn/sliew/scaleph/application/flink/resource/handler/FlinkImageMapping.java @@ -38,7 +38,7 @@ public enum FlinkImageMapping { SQL_1_17(FlinkJobType.SQL, OperatorFlinkVersion.v1_17, FlinkVersionMapping.V_1_17, "ghcr.io/flowerfine/scaleph-sql-template:1.17"), SQL_1_18(FlinkJobType.SQL, OperatorFlinkVersion.v1_18, FlinkVersionMapping.V_1_18, "ghcr.io/flowerfine/scaleph-sql-template:1.18"), - SEATUNNEL_1_16(FlinkJobType.SEATUNNEL, OperatorFlinkVersion.v1_16, FlinkVersionMapping.V_1_16, "ghcr.io/flowerfine/scaleph-seatunnel:2.3.7-flink-1.16"), + SEATUNNEL_1_16(FlinkJobType.SEATUNNEL, OperatorFlinkVersion.v1_16, FlinkVersionMapping.V_1_16, "ghcr.io/flowerfine/scaleph-seatunnel:2.3.8-flink-1.16"), FLINK_CDC_1_18(FlinkJobType.FLINK_CDC, OperatorFlinkVersion.v1_18, FlinkVersionMapping.V_1_18, "ghcr.io/flowerfine/scaleph-flink-cdc:3.0.0-flink-1.18"), ; diff --git a/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelVersion.java b/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelVersion.java index 2b8717bd3..82e1e35c0 100644 --- a/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelVersion.java +++ b/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelVersion.java @@ -33,6 +33,7 @@ public enum SeaTunnelVersion implements DictInstance { V_2_3_5("2.3.5", "2.3.5"), V_2_3_6("2.3.6", "2.3.6"), V_2_3_7("2.3.7", "2.3.7"), + V_2_3_8("2.3.8", "2.3.8"), ; @JsonCreator diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/ClickHouseProperties.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/ClickHouseProperties.java index 19ee230f1..ed15de275 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/ClickHouseProperties.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/ClickHouseProperties.java @@ -19,6 +19,7 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.clickhosue; import cn.sliew.scaleph.plugin.framework.property.*; +import org.codehaus.jackson.JsonNode; public enum ClickHouseProperties { ; @@ -60,4 +61,13 @@ public enum ClickHouseProperties { .addValidator(Validators.NON_BLANK_VALIDATOR) .validateAndBuild(); + public static final PropertyDescriptor CLICKHOUSE_CONFIG = new PropertyDescriptor.Builder() + .name("clickhouse.config") + .description( + "clickhouse.* The way to specify the parameter is to add the prefix clickhouse. to the original parameter name. For example, the way to specify socket_timeout is: clickhouse.socket_timeout = 50000 . " + + "If these non-essential parameters are not specified, they will use the default values given by clickhouse-jdbc.\n") + .type(PropertyType.OBJECT) + .parser(Parsers.JSON_PARSER) + .validateAndBuild(); + } diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/sink/ClickHouseSinkProperties.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/sink/ClickHouseSinkProperties.java index c44965533..3c11ebde8 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/sink/ClickHouseSinkProperties.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/sink/ClickHouseSinkProperties.java @@ -33,15 +33,6 @@ public enum ClickHouseSinkProperties { .addValidator(Validators.NON_BLANK_VALIDATOR) .validateAndBuild(); - public static final PropertyDescriptor CLICKHOUSE_CONFIG = new PropertyDescriptor.Builder() - .name("clickhouse.config") - .description( - "clickhouse.* The way to specify the parameter is to add the prefix clickhouse. to the original parameter name. For example, the way to specify socket_timeout is: clickhouse.socket_timeout = 50000 . " - + "If these non-essential parameters are not specified, they will use the default values given by clickhouse-jdbc.\n") - .type(PropertyType.OBJECT) - .parser(Parsers.JSON_PARSER) - .validateAndBuild(); - public static final PropertyDescriptor BULK_SIZE = new PropertyDescriptor.Builder() .name("bulk_size") .description( diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/source/ClickHouseSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/source/ClickHouseSourcePlugin.java index 4dd4c9e29..6fa9562d5 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/source/ClickHouseSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/clickhosue/source/ClickHouseSourcePlugin.java @@ -50,6 +50,7 @@ public ClickHouseSourcePlugin() { final List props = new ArrayList<>(); props.add(SQL); props.add(SERVER_TIME_ZONE); + props.add(CLICKHOUSE_CONFIG); props.add(CommonProperties.PARALLELISM); props.add(CommonProperties.RESULT_TABLE_NAME); supportedProperties = Collections.unmodifiableList(props); diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/source/DorisSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/source/DorisSourcePlugin.java index de705d811..6e4cc3bcf 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/source/DorisSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/source/DorisSourcePlugin.java @@ -37,6 +37,7 @@ import java.util.List; import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.doris.DorisProperties.*; +import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.doris.DorisProperties.DORIS_BATCH_SIZE; import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.doris.source.DorisSourceProperties.*; @AutoService(SeaTunnelConnectorPlugin.class) @@ -51,6 +52,7 @@ public DorisSourcePlugin() { props.add(TABLE); props.add(DORIS_READ_FIELD); props.add(DORIS_FILTER_QUERY); + props.add(DORIS_BATCH_SIZE); props.add(DORIS_REQUEST_CONNECT_TIMEOUT_MS); props.add(DORIS_REQUEST_QUERY_TIMEOUT_S); props.add(DORIS_REQUEST_READ_TIMEOUT_MS); diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourcePlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourcePlugin.java index 8c9d2278f..060f7e55d 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourcePlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourcePlugin.java @@ -47,8 +47,10 @@ public ElasticsearchSourcePlugin() { "Used to read data from Elasticsearch. support version >= 2.x and < 8.x.", ElasticsearchSourcePlugin.class.getName()); final List props = new ArrayList<>(); + props.add(INDEX_LIST); props.add(INDEX); props.add(SOURCE); + props.add(ARRAY_COLUMN); props.add(QUERY); props.add(SCROLL_TIME); props.add(SCROLL_SIZE); diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourceProperties.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourceProperties.java index b25aa7085..1d9b3f5a0 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourceProperties.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/source/ElasticsearchSourceProperties.java @@ -27,6 +27,14 @@ public enum ElasticsearchSourceProperties { ; + public static final PropertyDescriptor INDEX_LIST = new PropertyDescriptor.Builder() + .name("index_list") + .description("The index_list is used to define multi-index synchronization tasks") + .type(PropertyType.OBJECT) + .parser(Parsers.JSON_PARSER) + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); + public static final PropertyDescriptor SOURCE = new PropertyDescriptor.Builder() .name("source") .description("The fields of index.") @@ -35,6 +43,14 @@ public enum ElasticsearchSourceProperties { .addValidator(Validators.NON_BLANK_VALIDATOR) .validateAndBuild(); + public static final PropertyDescriptor ARRAY_COLUMN = new PropertyDescriptor.Builder() + .name("array_column") + .description("The fields of array type") + .type(PropertyType.STRING) + .parser(Parsers.STRING_PARSER) + .addValidator(Validators.NON_BLANK_VALIDATOR) + .validateAndBuild(); + public static final PropertyDescriptor QUERY = new PropertyDescriptor.Builder() .name("query") .description("Elasticsearch DSL. You can control the range of data read.") diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/email/sink/EmailSinkPlugin.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/email/sink/EmailSinkPlugin.java index eeb26fa17..f46b70b78 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/email/sink/EmailSinkPlugin.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/email/sink/EmailSinkPlugin.java @@ -43,6 +43,7 @@ public EmailSinkPlugin() { props.add(EMAIL_TRANSPORT_PROTOCOL); props.add(EMAIL_FROM_ADDRESS); props.add(EMAIL_SMTP_AUTH); + props.add(EMAIL_SMTP_PORT); props.add(EMAIL_AUTHORIZATION_CODE); props.add(EMAIL_TO_ADDRESS); props.add(EMAIL_MESSAGE_HEADLINE); diff --git a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/email/sink/EmailSinkProperties.java b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/email/sink/EmailSinkProperties.java index 96b7714a6..9fe17f721 100644 --- a/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/email/sink/EmailSinkProperties.java +++ b/scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/email/sink/EmailSinkProperties.java @@ -19,6 +19,7 @@ package cn.sliew.scaleph.plugin.seatunnel.flink.connectors.email.sink; import cn.sliew.scaleph.plugin.framework.property.*; +import com.amazonaws.services.dynamodbv2.xspec.BOOL; public enum EmailSinkProperties { ; @@ -50,13 +51,20 @@ public enum EmailSinkProperties { .addValidator(Validators.NON_BLANK_VALIDATOR) .validateAndBuild(); - public static final PropertyDescriptor EMAIL_SMTP_AUTH = new PropertyDescriptor.Builder() + public static final PropertyDescriptor EMAIL_SMTP_AUTH = new PropertyDescriptor.Builder() .name("email_smtp_auth") .description("Whether to authenticate the customer") - .type(PropertyType.STRING) - .parser(Parsers.STRING_PARSER) - .properties(Property.Required) - .addValidator(Validators.NON_BLANK_VALIDATOR) + .type(PropertyType.BOOLEAN) + .parser(Parsers.BOOLEAN_PARSER) + .addValidator(Validators.BOOLEAN_VALIDATOR) + .validateAndBuild(); + + public static final PropertyDescriptor EMAIL_SMTP_PORT = new PropertyDescriptor.Builder() + .name("email_smtp_port") + .description("Select port for authentication.") + .type(PropertyType.INT) + .parser(Parsers.INTEGER_PARSER) + .addValidator(Validators.INTEGER_VALIDATOR) .validateAndBuild(); public static final PropertyDescriptor EMAIL_AUTHORIZATION_CODE = new PropertyDescriptor.Builder() diff --git a/scaleph-ui-react/src/locales/zh-CN/pages/project.ts b/scaleph-ui-react/src/locales/zh-CN/pages/project.ts index 72dfefc89..d0f703fe6 100644 --- a/scaleph-ui-react/src/locales/zh-CN/pages/project.ts +++ b/scaleph-ui-react/src/locales/zh-CN/pages/project.ts @@ -334,6 +334,7 @@ export default { 'pages.project.di.step.email.emailTransportProtocol': '邮件传输协议', 'pages.project.di.step.email.emailFromAddress': '邮件发送者地址', 'pages.project.di.step.email.emailSmtpAuth': '启用SMTP认证', + 'pages.project.di.step.email.emailSmtpPort': 'SMTP端口号', 'pages.project.di.step.email.emailAuthorizationCode': 'SMTP认证代码', 'pages.project.di.step.email.emailToAddress': '邮件接收者地址', 'pages.project.di.step.email.emailMessageHeadline': '邮件标题', @@ -611,6 +612,8 @@ export default { 'pages.project.di.step.elasticsearch.scrollSize': '滚动大小', 'pages.project.di.step.elasticsearch.source': '字段列表', 'pages.project.di.step.elasticsearch.source.tooltip': '字段列表', + 'pages.project.di.step.elasticsearch.arrayColumn': '数组字段', + 'pages.project.di.step.elasticsearch.arrayColumn.placeholder': '{tags = "array",phones = "array"}', // neo4j 'pages.project.di.step.neo4j.uri': 'URI', @@ -679,7 +682,7 @@ export default { 'pages.project.di.step.doris.sinkMaxRetries': '重试次数', 'pages.project.di.step.doris.sinkBufferSize': 'Buffer 大小', 'pages.project.di.step.doris.sinkBufferCount': 'Buffer 数量', - 'pages.project.di.step.doris.dorisBatchSize': '批量写入数量', + 'pages.project.di.step.doris.dorisBatchSize': '批量读取或写入数量', 'pages.project.di.step.doris.needsUnsupportedTypeCasting': '启用类型转换', 'pages.project.di.step.doris.needsUnsupportedTypeCasting.tooltip': '上游数据类型和 Doris 数据类型不匹配时，将上游数据转换成 Doris 支持的数据类型。如 Decimal64 转换成 Double', 'pages.project.di.step.doris.saveModeCreateTemplate': 'Schema 创建模板', diff --git a/scaleph-ui-react/src/pages/Project/Workspace/DataIntegration/SeaTunnel/Dag/components/node/canvas-node.tsx b/scaleph-ui-react/src/pages/Project/Workspace/DataIntegration/SeaTunnel/Dag/components/node/canvas-node.tsx index 283bc7001..1c0928337 100644 --- a/scaleph-ui-react/src/pages/Project/Workspace/DataIntegration/SeaTunnel/Dag/components/node/canvas-node.tsx +++ b/scaleph-ui-react/src/pages/Project/Workspace/DataIntegration/SeaTunnel/Dag/components/node/canvas-node.tsx @@ -64,7 +64,7 @@ const SeaTunnelConnectorDagNode = ({node}: { node: Node }) => { > {node.data.label} - +