diff --git a/README-ZH.md b/README-ZH.md index 736279157..19e680fc0 100644 --- a/README-ZH.md +++ b/README-ZH.md @@ -53,8 +53,8 @@ Exchangis 抽象了一套统一的数据源和同步作业定义插件，允许 ## 相关文档 -[安装部署文档](https://github.com/WeDataSphere/Exchangis/blob/dev-1.0.0-rc/docs/zh_CN/ch1/exchangis_deploy_cn.md) -[用户手册](https://github.com/WeDataSphere/Exchangis/blob/dev-1.0.0-rc/docs/zh_CN/ch1/exchangis_user_manual_cn.md) +[安装部署文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_deploy_cn.md) +[用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_user_manual_cn.md) ## 交流贡献 diff --git a/README.md b/README.md index 2c8975105..8bd867ded 100644 --- a/README.md +++ b/README.md @@ -52,8 +52,8 @@ With the help of [Linkis](https://github.com/apache/incubator-linkis) computing ## Documents -[Quick Deploy](https://github.com/WeDataSphere/Exchangis/blob/dev-1.0.0-rc/docs/zh_CN/ch1/exchangis_deploy_cn.md) -[User Manual](https://github.com/WeDataSphere/Exchangis/blob/dev-1.0.0-rc/docs/zh_CN/ch1/exchangis_user_manual_cn.md) +[Quick Deploy](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/en_US/ch1/exchangis_deploy_en.md) +[User Manual](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/en_US/ch1/exchangis_user_manual_en.md) ## Communication and contribution diff --git a/assembly-package/config/application-exchangis.yml b/assembly-package/config/application-exchangis.yml index f7247d4aa..86268b937 100644 --- a/assembly-package/config/application-exchangis.yml +++ b/assembly-package/config/application-exchangis.yml @@ -6,7 +6,7 @@ spring: eureka: client: serviceUrl: - defaultZone: http://127.0.0.1:3306/eureka/ + defaultZone: http://{IP}:{PORT}/eureka/ instance: metadata-map: test: wedatasphere diff --git a/assembly-package/config/config.sh b/assembly-package/config/config.sh index e65884fa2..9a4a0e502 100644 --- a/assembly-package/config/config.sh +++ b/assembly-package/config/config.sh @@ -1,4 +1,11 @@ -LINKIS_GATEWAY_HOST= -LINKIS_GATEWAY_PORT= -EXCHANGIS_PORT= -EUREKA_URL= \ No newline at end of file +#LINKIS_GATEWAY服务地址IP，用于查找linkis-mg-gateway服务 +LINKIS_GATEWAY_HOST={IP} + +#LINKIS_GATEWAY服务地址端口，用于查找linkis-mg-gateway服务 +LINKIS_GATEWAY_PORT={PORT} + +#Exchangis服务端口 +EXCHANGIS_PORT={PORT} + +#Eureka服务URL +EUREKA_URL=http://{IP:PORT}/eureka/ \ No newline at end of file diff --git a/assembly-package/config/db.sh b/assembly-package/config/db.sh index b86d3361d..cf33388e3 100644 --- a/assembly-package/config/db.sh +++ b/assembly-package/config/db.sh @@ -1,9 +1,9 @@ # 设置数据库的连接信息 # 包括IP地址、数据库名称、用户名、端口 -MYSQL_HOST= -MYSQL_PORT= -MYSQL_USERNAME= -MYSQL_PASSWORD= -DATABASE= +MYSQL_HOST={IP} +MYSQL_PORT={PORT} +MYSQL_USERNAME={username} +MYSQL_PASSWORD={password} +DATABASE={dbName} diff --git a/assembly-package/config/exchangis-server.properties b/assembly-package/config/exchangis-server.properties index dc55c3f9b..dfba5255e 100644 --- a/assembly-package/config/exchangis-server.properties +++ b/assembly-package/config/exchangis-server.properties @@ -15,33 +15,25 @@ # # -#wds.linkis.test.mode=true -wds.linkis.test.mode=false +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://{IP}:{PORT}/{database}?useSSL=false&characterEncoding=UTF-8&allowMultiQueries=true -wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/exchangis?useSSL=false&characterEncoding=UTF-8&allowMultiQueries=true +wds.linkis.server.mybatis.datasource.username={username} -wds.linkis.server.mybatis.datasource.username=username - -wds.linkis.server.mybatis.datasource.password=password - -wds.linkis.gateway.ip=127.0.0.1 -wds.linkis.gateway.port=9001 -wds.linkis.gateway.url=http://127.0.0.1:9001/ +wds.linkis.server.mybatis.datasource.password={password} wds.linkis.log.clear=true wds.linkis.server.version=v1 -## datasource client -wds.exchangis.datasource.client.serverurl=http://127.0.0.1:9001 +# datasource client +wds.exchangis.datasource.client.serverurl=http://{LINKIS_IP}:{LINKIS_PORT}/ wds.exchangis.datasource.client.authtoken.key=EXCHANGIS-AUTH wds.exchangis.datasource.client.authtoken.value=EXCHANGIS-AUTH wds.exchangis.datasource.client.dws.version=v1 # launcher client -wds.exchangis.client.linkis.server-url=http://127.0.0.1:9001 +wds.exchangis.client.linkis.server-url=http://{LINKIS_IP}:{LINKIS_PORT}/ wds.exchangis.client.linkis.token.value=EXCHANGIS-AUTH - wds.exchangis.datasource.extension.dir=exchangis-extds ##restful @@ -65,5 +57,5 @@ wds.exchangis.job.task.scheduler.load-balancer.flexible.segments.min-occupy=0.25 wds.exchangis.job.task.scheduler.load-balancer.flexible.segments.max-occupy=0.5 #wds.exchangis.job.scheduler.group.max.running-jobs=4 -wds.linkis.session.ticket.key=bdp-user-ticket-id +wds.linkis-session.ticket.key=bdp-user-ticket-id diff --git a/assembly-package/sbin/launcher.sh b/assembly-package/sbin/launcher.sh index ba9456329..60bc62364 100644 --- a/assembly-package/sbin/launcher.sh +++ b/assembly-package/sbin/launcher.sh @@ -32,6 +32,8 @@ DEBUG_PORT="7006" SPRING_PROFILE="exchangis" SLEEP_TIMEREVAL_S=2 +CONF_PATH=${DIR}/../config + function LOG(){ currentTime=`date "+%Y-%m-%d %H:%M:%S.%3N"` echo -e "$currentTime [${1}] ($$) $2" | tee -a ${SHELL_LOG} @@ -212,7 +214,9 @@ launcher_start(){ wait_for_startup 20 $1 $2 if [[ $? -eq 0 ]]; then LOG INFO "Launcher: [ $1 ] start success" - LOG INFO "Please check exchangis server in EUREKA_ADDRESS: ${EUREKA_URL} " + APPLICATION_YML="${CONF_PATH}/application-exchangis.yml" + EUREKA_URL=`cat ${APPLICATION_YML} | grep Zone | sed -n '1p'` + LOG INFO "Please check exchangis server in EUREKA_ADDRESS: ${EUREKA_URL#*:} " else LOG ERROR "Launcher: [ $1 ] start fail over 20 seconds, please retry it" fi diff --git a/db/exchangis_ddl.sql b/db/exchangis_ddl.sql index 5e0671b22..04071533d 100644 --- a/db/exchangis_ddl.sql +++ b/db/exchangis_ddl.sql @@ -81,6 +81,16 @@ CREATE TABLE `exchangis_project_info` ( PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1497870871035973934 DEFAULT CHARSET=utf8; +-- exchangis_v4.exchangis_project_user definition +CREATE TABLE `exchangis_project_user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `project_id` bigint(20) NOT NULL, + `priv_user` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `priv` int(20) DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT + -- exchangis_v4.exchangis_launchable_task definition DROP TABLE IF EXISTS `exchangis_launchable_task`; CREATE TABLE `exchangis_launchable_task` ( diff --git a/docs/en_US/ch1/exchangis_deploy_en.md b/docs/en_US/ch1/exchangis_deploy_en.md index 27eaedd56..aba27e647 100644 --- a/docs/en_US/ch1/exchangis_deploy_en.md +++ b/docs/en_US/ch1/exchangis_deploy_en.md @@ -28,7 +28,7 @@ $\color{#FF0000}{Note: be sure to reinstall dss1.0.1, and the linkis version mus [linkis1.1.1 code address ](https://github.com/apache/incubator-linkis/tree/release-1.1.1) -[DSS1.0.1 code address ](https://github.com/WeBankFinTech/DataSphereStudio/tree/master) +[DSS1.0.1 code address ](https://github.com/WeBankFinTech/DataSphereStudio/tree/dev-1.1.0) datasource enabled @@ -63,7 +63,7 @@ Please ensure that DSS1.0.1 and Linkis1.1.0 are basically available. HiveQL scri #### 2.1.1 Download binary package -Download the latest installation package from the Released release of Exchangis [click to jump to the release interface](https://github.com/WeBankFinTech/Exchangis/releases). +Download the latest installation package from the Released release of Exchangis [click to jump to the release interface](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.0.0). #### 2.1.2 Compile and package @@ -187,7 +187,7 @@ As shown in the figure below: #### 2.7.1 Get the front-end installation package -Exchangis has provided compiled front-end installation package by default, which can be downloaded and used directly ：[Click to jump to the Release interface](https://github.com/WeBankFinTech/Exchangis/releases) +Exchangis has provided compiled front-end installation package by default, which can be downloaded and used directly ：[Click to jump to the Release interface](https://github.com/WeBankFinTech/Exchangis/releases/download/release-1.0.0/web-dist.zip) You can also compile the exchange front-end by yourself and execute the following command in the exchanise root directory: @@ -269,12 +269,12 @@ Please visit the Exchange front-end page at http://${EXCHANGIS_INSTALL_IP}:8098 ## 3. DSS ExchangisAppConn installation and deployment -If you want to use Exchangis1.0.1 front-end, you also need to install the DSS ExchangisAppConn plugin. Please refer to: [ExchangisAppConn installation documentation for plugins ](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/en_US/ch1/exchangis_appconn_deploy_en.md) +If you want to use Exchangis1.0.1 front-end, you also need to install the DSS ExchangisAppConn plugin. Please refer to: [ExchangisAppConn installation documentation for plugins ](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/en_US/ch1/exchangis_appconn_deploy_en.md) ## 4. Linkis Sqoop engine installation and deployment -If you want to execute the Sqoop operation of Exchangis1.0.1 normally, you also need to install the Linkis Sqoop engine. Please refer to: : [Linkis Sqoop engine installation documentation ](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/en_US/ch1/exchangis_sqoop_deploy_en.md) +If you want to execute the Sqoop operation of Exchangis1.0.1 normally, you also need to install the Linkis Sqoop engine. Please refer to: : [Linkis Sqoop engine installation documentation ](https://linkis.staged.apache.org/zh-CN/docs/1.1.2/engine_usage/sqoop) ## 5. How to log in and use Exchangis - To be supplemented ! +Exchangis1.0 for more instructions, please refer to the user manual.[Exchangis1.0 user manual](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/en_US/ch1/exchangis_user_manual_en.md) diff --git a/docs/en_US/ch1/exchangis_interface_en.md b/docs/en_US/ch1/exchangis_interface_en.md new file mode 100644 index 000000000..019f4f2aa --- /dev/null +++ b/docs/en_US/ch1/exchangis_interface_en.md @@ -0,0 +1,839 @@ +# Exchangis interface document + +## Exchangis datasource module + +### 1、Get datasource type + +Interface description：Get the datasource type according to the information of request + +Request URL：/dss/exchangis/main/datasources/type + +Request mode：GET + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | ------- | +| request | HttpServletRequest | yes | / | request | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------ | ----------- | ------------- | ---------------------------- | +| method | String | yes | | Called method (request path) | +| status | int | yes | | Response status code | +| message | String | no | | Information of the response | +| data | List | yes | | The returned data | + +### 2、Query datasource + +Interface description：Query the required datasource according to vo + +Request URL：/dss/exchangis/main/datasources/query + +Request mode：GET、POST + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | ------- | +| request | HttpServletRequest | yes | / | request | +| vo | DataSourceQueryVO | yes | / | | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------ | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | List | yes | / | The returned data | + +### 3、Query datasource + +Interface description：Query datasource according to request information + +Request URL：/dss/exchangis/main/datasources + +Request mode：GET + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| -------- | ------------------ | ----------- | ------------- | ------------------- | +| request | HttpServletRequest | yes | / | request | +| typeId | Long | yes | / | datasource typeId | +| typeName | String | yes | / | datasource Typename | +| page | Integer | yes | / | page num | +| size | Integer | yes | / | size per page | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | --------------------- | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | List\ | yes | / | The returned data | + +### 4、Query datasource keydefines + +Interface description：Query key definitions of datasource according to datasource type ID + +Request URL：/dss/exchangis/main/datasources/types/{dataSourceTypeId}/keydefines + +Request mode：GET + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ---------------- | ------------------ | ----------- | ------------- | ----------------- | +| request | HttpServletRequest | yes | / | | +| dataSourceTypeId | Long | yes | / | dataSource typeId | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ---------------------- | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | List[Map[String, Any]] | yes | / | The returned data | + +### 5、Get datasource version + +Interface description：Get the datasource version according to the datasource ID + +Request URL：/dss/exchangis/main/datasources/{id}/versions + +Request mode：GET + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | ------------- | +| request | HttpServletRequest | yes | / | request | +| id | Long | yes | / | datasource id | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------------- | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | List> | yes | / | The returned data | + +### 6、Create datasource + +Interface description：Create a datasource according to the DataSourceCreateVO + +Request URL：/dss/exchangis/main/datasources + +Request mode：POST + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------------------ | ------------------ | ----------- | ------------- | ------------------ | +| request | HttpServletRequest | yes | / | request | +| dataSourceCreateVO | DataSourceCreateVO | yes | / | dataSourceCreateVO | +| bindingResult | BindingResult | yes | / | bindingResult | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------ | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | Long | yes | / | The returned data | + +### 7、Get datasource information + +Interface description：Get datasource information according to request information, datasource ID and version ID + +Request URL：/dss/exchangis/main/datasources/{id} + +Request mode：GET + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| --------- | ------------------ | ----------- | ------------- | ------------- | +| request | HttpServletRequest | yes | / | request | +| id | Long | yes | / | datasource id | +| versionId | String | yes | / | version id | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ----------------- | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | DataSourceItemDTO | yes | / | The returned data | + +### 8、Get datasource connect params + +Interface description：Acquiring datasource connection parameters according to the request information and the datasource ID + +Request URL：/dss/exchangis/main/datasources/{id}/connect_params + +Request mode：GET + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | ------------- | +| request | HttpServletRequest | yes | / | request | +| Long | id | yes | / | datasource id | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | -------------------------- | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | java.util.Map[String, Any] | yes | / | The returned data | + +### 9、Update datasource + +Interface description：Update the datasource according to the request information, datasource type ID and DataSourceCreateVO + +Request URL：/dss/exchangis/main/datasources/{id} + +Request mode：PUT + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------------------ | ------------------ | ----------- | ------------- | ------------------ | +| request | HttpServletRequest | yes | / | request | +| id | Long | yes | / | datasource id | +| dataSourceCreateVO | DataSourceCreateVO | yes | / | dataSourceCreateVO | +| bindingResult | BindingResult | yes | / | bindingResult | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------ | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | Map | yes | / | The returned data | + +### 10、Publish datasource + +Interface description：Publish the datasource according to the request request, datasource ID and version ID + +Request URL：/dss/exchangis/main/datasources/{id}/{version}/publish + +Request mode：PUT + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | --------------------- | +| request | HttpServletRequest | yes | / | request | +| id | Long | yes | / | datasource id | +| version | Long | yes | / | datasource version id | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------ | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | Map | yes | / | The returned data | + +### 11、Expire datasource + +Interface description：Expire datasource according to request information and datasource ID + +Request URL：/dss/exchangis/main/datasources/{id}/expire + +Request mode：PUT + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | ------------- | +| request | HttpServletRequest | yes | / | request | +| id | Long | yes | / | datasource id | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------ | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | Map | yes | / | The returned data | + +### 12、Connect datasource + +Interface description：Connect the datasource according to the request information, datasource ID and version ID + +Request URL：/dss/exchangis/main/datasources/{id}/{version}/connect + +Request mode：PUT + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | --------------------- | +| request | HttpServletRequest | yes | / | request | +| id | Long | yes | / | datasource id | +| version | Long | yes | / | datasource version id | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------ | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | Map | yes | / | The returned data | + +### 13、Connect datasource + +Interface description：Connect the datasource according to the request information and the datasource ID + +Request URL：/dss/exchangis/main/datasources/op/connect + +Request mode：POST + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | ------------- | +| request | HttpServletRequest | yes | / | request | +| id | Long | yes | / | datasource id | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------ | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | Map | yes | / | The returned data | + +### 14、Delete datasource + +Interface description：Connect the datasource according to the request information and the datasource type ID + +Request URL：/dss/exchangis/main/datasources/{id} + +Request mode：DELETE + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ---------------- | ------------------ | ----------- | ------------- | ----------------- | +| request | HttpServletRequest | yes | / | request | +| dataSourceTypeId | Long | yes | / | dataSource typeId | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------ | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | Long | yes | / | The returned data | + +### 15、Query datasource by datasourceId + +Interface description：Query the datasource according to the request information, datasource type and datasource ID + +Request URL：/dss/exchangis/main/datasources/{type}/{id}/dbs + +Request mode：GET + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | --------------- | +| request | HttpServletRequest | yes | / | request | +| type | String | yes | / | datasource type | +| id | Long | yes | / | datasource id | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | -------------- | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | List\ | yes | / | The returned data | + +### 16、Get table by datasourceId and datasourceName + +Interface description：According to the request information, datasource type, datasource ID and database, a data table is obtained. + +Request URL：/dss/exchangis/main/datasources/{type}/{id}/dbs/{dbName}/tables + +Request mode：GET + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | -------------- | +| request | HttpServletRequest | yes | / | request | +| type | String | yes | / | datsource type | +| id | Long | yes | / | datasource id | +| dbName | String | yes | / | database name | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | -------------- | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | List\ | yes | / | The returned data | + +### 17、Get table field by datasourceId、datasourceName and table + +Interface description：Get table fields according to request information, datasource type, datasource ID, database name and data table. + +Request URL：/dss/exchangis/main/datasources/{type}/{id}/dbs/{dbName}/tables/{tableName}/fields + +Request mode：GET + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| --------- | ------------------ | ----------- | ------------- | --------------- | +| request | HttpServletRequest | yes | / | request | +| type | String | yes | / | datasource type | +| id | Long | yes | / | datasource id | +| dbName | String | yes | / | database name | +| tableName | String | yes | / | table name | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ---------------------------------- | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | List\ | yes | / | The returned data | + +### 18、Get fieldsmapping + +Interface description：Acquiring table field list information according to the request information and field mapping VO + +Request URL：/dss/exchangis/main/datasources/fieldsmapping + +Request mode：POST + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | -------------- | +| request | HttpServletRequest | yes | / | request | +| vo | FieldMappingVO | yes | / | FieldMappingVO | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------------- | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | List> | yes | / | The returned data | + +### 19、Get params by datasourceType + +Interface description：Get parameters according to request information, engine, datasource type and file system path + +Request URL：/dss/exchangis/main/datasources/{engine}/{type}/params/ui + +Request mode：POST + +Request parameters： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | --------------- | +| request | HttpServletRequest | yes | / | request | +| engine | String | yes | / | enigne | +| type | String | yes | / | datasource type | +| dir | String | yes | / | dir | + +Return parameter： + +| Name | Type | If required | Default value | Remark | +| ------- | ------------------ | ----------- | ------------- | ---------------------------- | +| method | String | yes | / | Called method (request path) | +| status | int | yes | / | Response status code | +| message | String | no | / | Information of the response | +| data | List> | yes | / | The returned data | + + + +## Exchangis job execution module + +### 1、Submit the configured job for execution. + +Interface description ： Submit ExchangisJob and return jobExecutionId in the background. + +Request URL：/api/rest_j/v1/exchangis/job/{id}/execute + +Request mode：POST + +Request parameters： + +| Name | Type | Remark | If required | Default value | +| --------------------- | ------- | ------------------------------------------------------------ | ----------- | ------------- | +| id | Long | Exchangis's ID | yes | / | +| permitPartialFailures | boolean | Whether partial failure is allowed. If true, even if some subtasks fail, the whole Job will continue to execute. After the execution is completed, the Job status is Partial_Success. This parameter is the requestBody parameter. | no | false | + +Return parameter ： + +| Name | Type | Remark | If required | Default value | +| -------------- | ------ | ------------------------------------ | ----------- | ------------- | +| method | String | Called method (request path) | yes | / | +| status | int | Response status code | yes | / | +| message | String | Information of the response | no | / | +| data | Map | The returned data | yes | / | +| jobExecutionId | String | Returns the execution log of the Job | yes | / | + +Back to example ： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/{id}/execute", + "status": 0, + "message": "Submitted succeed(Submit successfully)！", + "data": { + "jobExecutionId": "555node1node2node3execId1" +} + +``` + +### 2、Get the execution status of Job + +Interface description： Get the status of Job according to jobExecutionId + +Request URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/status + +Request mode：GET + +Request parameters： + +| Name | Type | Remark | If required | Default value | +| -------------- | ------ | ---------------------------- | ----------- | ------------- | +| jobExecutionId | String | Execution ID of ExchangisJob | yes | / | + +Back to example ： + +| Name | Type | Remark | If required | Default value | +| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | +| method | String | Called method (request path) | yes | / | +| status | int | Response status code | yes | / | +| message | String | Information of the response | no | / | +| data | Map | The returned data | yes | / | +| jobExecutionId | String | The status of the executed job, including: initiated, Scheduled, Running, WaitForRetry, Cancelled, Failed, Partial_Success, Success, Undefined, Timeout. Among them, the Running state indicates that it is running, and all of them are completed since Cancelled | yes | / | + +Back to example ： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/execution/{id}/status", + "status": 0, + "message": "Submitted succeed(Submit successfully)！", + "data": { + "status": "Running", + "progress": 0.1 +} +``` + +### 3、Get the task list executed by this Job + +Interface description：Get the task list through jobExecutionId + +Prerequisite: the task list can only be obtained after the Job's execution status is Running, otherwise the returned task list is empty. + +Request URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/tasklist + +Request mode：GET + +Request parameters： + +| Name | Type | Remark | If required | Default value | +| -------------- | ------ | ------ | ----------- | ------------- | +| jobExecutionId | String | string | yes | / | + +Back to example ： + +| Name | Type | Remark | If required | Default value | +| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | +| method | String | Called method (request path) | yes | / | +| status | int | Response status code | yes | / | +| message | String | Information of the response | no | / | +| data | Map | The returned data | yes | / | +| jobExecutionId | String | Task list. The execution status of the Job must be Running before you can get the task list, otherwise the returned task list is empty. Please note: task has no Partial_Success status | yes | / | + +Back to example ： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/tasklist", + "status": 0, + "message": "Submitted succeed(Submit successfully)！", + "data": { + "tasks": [ + { + "taskId": 5, + "name": "test-1", + "status": "Inited", // There is no task Partial_Success status. + "createTime": "2022-01-03 09:00:00", + "launchTime": null, + "lastUpdateTime": "2022-01-03 09:00:00", + "engineType": "sqoop", + "linkisJobId": null, + "linkisJobInfo": null, + "executeUser": "enjoyyin" + } + ] + } +} + +``` + +### 4、Get the execution progress of Job & task + +Interface description： Get the execution progress through jobExecutionId + +Prerequisites: the execution status of the Job must be Running before you can get the progress of the task list. + +Request URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/progress + +Request mode：GET + +Request parameters： + +| Name | Type | Remark | If required | Default value | +| -------------- | ------ | ---------------------------- | ----------- | ------------- | +| jobExecutionId | String | Execution ID of ExchangisJob | yes | / | + +Back to example ： + +| Name | Type | Remark | If required | Default value | +| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | +| method | String | Called method (request path) | yes | / | +| status | int | Response status code | yes | / | +| message | String | Information of the response | no | / | +| data | Map | The returned data | yes | / | +| jobExecutionId | String | Task list. The execution status of the Job must be Running before you can get the task list, otherwise the returned task list is empty | yes | / | + +Back to example ： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/progress", + "status": 0, + "message": "Submitted succeed(Submit successfully)！", + "data": { + "job": { + "status": "Running", + "progress": 0.1, + "tasks": { + "running": [ + { + "taskId": 5, + "name": "test-1", + "status": "Running", + "progress": 0.1 + } + ], + "Inited": [ + { + "taskId": 5, + "name": "test-1", + "status": "Inited", + "progress": 0.1 + } + ], + "Scheduled": [], + "Success": [], + "Failed": [], // If there is a Failed task, the Job will fail directly. + "WaitForRetry": [], + "Cancelled": [], // If there is a Cancelled task, the Job will fail directly + "Undefined": [], // If there is a Undefined task, the Job will fail directly + "Timeout": [] + } + } + } +} + +``` + +### 5、Get the indicator information of task runtime + +Interface description：Through jobExecutionId and taskId, we can get the information of various indicators when task is running. + +Prerequisites: before you can get the indicator information of the task, the execution status of the task must be Running; otherwise, the returned information is empty. + +Request URL：/api/rest_j/v1/exchangis/task/execution/{taskId}/metrics + +Request mode：POST + +Request parameters： + +| Name | Type | Remark | If required | Default value | +| -------------- | ------ | --------------------------------------------------- | ----------- | ------------- | +| jobExecutionId | String | Execution ID of ExchangisJob，put it in requestBody | yes | / | +| taskId | String | Execution ID of task，put it in URI | yes | / | + +Back to example ： + +| Name | Type | Remark | If required | Default value | +| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | +| method | String | Called method (request path) | yes | / | +| status | int | Response status code | yes | / | +| message | String | Information of the response | no | / | +| data | Map | The returned data | yes | / | +| jobExecutionId | String | Information of each task index. The execution status of task must be Running before you can get the indicator information of task | yes | / | + +Back to example ： + +```json +{ + "method": "/api/rest_j/v1/exchangis/task/execution/{taskId}/metrics", + "status": 0, + "message": "Submitted succeed(Submit successfully)！", + "data": { + "task": { + "taskId": 5, + "name": "test-1", + "status": "running", + "metrics": { + "resourceUsed": { + "cpu": 10, // Unit：vcores + "memory": 20 // Unit：GB + }, + "traffic": { + "source": "mysql", + "sink": "hive", + "flow": 100 // Unit：Records/S + }, + "indicator": { + "exchangedRecords": 109345, // Unit：Records + "errorRecords": 5, + "ignoredRecords": 5 + } + } + } + } +} +``` + +### 6、Get the real-time log of Job + +Interface description：Get the real-time log of Job through jobExecutionId. + +Request URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/log? fromLine=&pageSize=&ignoreKeywords=&onlyKeywords=&lastRows= + +Request mode：GET + +Request parameters： + +| Name | Type | Remark | If required | Default value | +| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | +| jobExecutionId | String | Execution ID of ExchangisJob | yes | / | +| fromLine | int | Read the starting line of | no | 0 | +| pageSize | int | Read the number of log lines this time | no | 100 | +| ignoreKeywords | String | Ignore which keywords are in the line, and multiple keywords are separated by English | no | / | +| onlyKeywords | String | Only the lines where keywords are located are selected, and multiple keywords are separated in English | no | / | +| lastRows | int | Read only the last few lines of the log, which is equivalent to tail -f log. When this parameter is greater than 0, all the above parameters will be invalid | no | / | + +Back to example ： + +| Name | Type | Remark | If required | Default value | +| ------- | ------- | ------------------------------------------------------------ | ----------- | ------------- | +| method | String | Called method (request path) | yes | / | +| status | int | Response status code | yes | / | +| message | String | Information of the response | no | / | +| data | Map | The returned data | yes | / | +| endLine | int | The end line number of this reading, you can continue reading the log from endLine+1 next time | yes | / | +| isEnd | boolean | Have all the logs been read | no | / | +| logs | List | Returns the execution log of the Job | yes | / | + +Back to example ： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/log", + "status": 0, + "message": "Submitted succeed(Submit successfully)！", + "data": { + "endLine": 99, // The end line number of this reading, you can continue reading the log from endLine+1 next time + "isEnd": false, // Have all the logs been read + "logs": [ + "all": "", + "error": "", + "warn": "", + "info": "" + ] + } +} +``` + +### 7、Get the real-time log of task + +Interface description： Get the real-time log of task through jobExecutionId and taskId. + +Request URL：/api/rest_j/v1/exchangis/task/execution/{taskId}/log? jobExecutionId=&fromLine=&pageSize=&ignoreKeywords=&onlyKeywords=&lastRows= + +Request mode：GET + +Request parameters： + +| Name | Type | Remark | If required | Default value | +| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | +| taskId | String | Execution ID of task | yes | / | +| jobExecutionId | String | Execution ID of ExchangisJob | yes | / | +| fromLine | int | Read the starting line of | no | 0 | +| pageSize | int | Read the number of log lines this time | no | 100 | +| ignoreKeywords | String | Ignore which keywords are in the line, and multiple keywords are separated by English | no | / | +| onlyKeywords | String | Only the lines where keywords are located are selected, and multiple keywords are separated in English | no | / | +| lastRows | int | Read only the last few lines of the log, which is equivalent to tail -f log. When this parameter is greater than 0, all the above parameters will be invalid | no | / | + +Back to example ： + +| Name | Type | Remark | If required | Default value | +| ------- | ------- | ------------------------------------------------------------ | ----------- | ------------- | +| method | String | Called method (request path) | yes | / | +| status | int | Response status code | yes | / | +| message | String | Information of the response | no | / | +| data | Map | The returned data | yes | / | +| endLine | int | The end line number of this reading, you can continue reading the log from endLine+1 next time. | yes | / | +| isEnd | boolean | Have all the logs been read | yes | / | +| logs | List | Returns the execution log of the Job | yes | / | + +Back to example ： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/execution/{taskId}/log", + "status": 0, + "message": "Submitted succeed(Submit successfully)！", + "data": { + "endLine": 99, // The end line number of this reading, you can continue reading the log from endLine+1 next time. + "isEnd": false, // Have all the logs been read + "logs": [ + "all": "", + "error": "", + "warn": "", + "info": "" + ] + } +} +``` + diff --git a/docs/en_US/ch1/exchangis_job_execute_interface_en.md b/docs/en_US/ch1/exchangis_job_execute_interface_en.md deleted file mode 100644 index 27be74f00..000000000 --- a/docs/en_US/ch1/exchangis_job_execute_interface_en.md +++ /dev/null @@ -1,361 +0,0 @@ -# Exchangis job execution module interface document - -### 1、Submit the configured job for execution. - -Interface description ： Submit ExchangisJob and return jobExecutionId in the background. - -Request URL：/api/rest_j/v1/exchangis/job/{id}/execute - -Request mode：POST - -Request parameters： - -| Name | Type | Remark | If required | Default value | -| --------------------- | ------- | ------------------------------------------------------------ | ----------- | ------------- | -| id | Long | Exchangis's ID | yes | / | -| permitPartialFailures | boolean | Whether partial failure is allowed. If true, even if some subtasks fail, the whole Job will continue to execute. After the execution is completed, the Job status is Partial_Success. This parameter is the requestBody parameter. | no | false | - -Return parameter ： - -| Name | Type | Remark | If required | Default value | -| -------------- | ------ | ------------------------------------ | ----------- | ------------- | -| method | String | Called method (request path) | yes | / | -| status | int | Response status code | yes | / | -| message | String | Information of the response | no | / | -| data | Map | The returned data | yes | / | -| jobExecutionId | String | Returns the execution log of the Job | yes | / | - -Back to example ： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/{id}/execute", - "status": 0, - "message": "Submitted succeed(Submit successfully)！", - "data": { - "jobExecutionId": "555node1node2node3execId1" -} - -``` - -### 2、Get the execution status of Job - -Interface description： Get the status of Job according to jobExecutionId - -Request URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/status - -Request mode：GET - -Request parameters： - -| Name | Type | Remark | If required | Default value | -| -------------- | ------ | ---------------------------- | ----------- | ------------- | -| jobExecutionId | String | Execution ID of ExchangisJob | yes | / | - -Back to example ： - -| Name | Type | Remark | If required | Default value | -| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | -| method | String | Called method (request path) | yes | / | -| status | int | Response status code | yes | / | -| message | String | Information of the response | no | / | -| data | Map | The returned data | yes | / | -| jobExecutionId | String | The status of the executed job, including: initiated, Scheduled, Running, WaitForRetry, Cancelled, Failed, Partial_Success, Success, Undefined, Timeout. Among them, the Running state indicates that it is running, and all of them are completed since Cancelled | yes | / | - -Back to example ： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/execution/{id}/status", - "status": 0, - "message": "Submitted succeed(Submit successfully)！", - "data": { - "status": "Running", - "progress": 0.1 -} -``` - -### 3、Get the task list executed by this Job - -Interface description：Get the task list through jobExecutionId - -Prerequisite: the task list can only be obtained after the Job's execution status is Running, otherwise the returned task list is empty. - -Request URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/tasklist - -Request mode：GET - -Request parameters： - -| Name | Type | Remark | If required | Default value | -| -------------- | ------ | ------ | ----------- | ------------- | -| jobExecutionId | String | string | yes | / | - -Back to example ： - -| Name | Type | Remark | If required | Default value | -| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | -| method | String | Called method (request path) | yes | / | -| status | int | Response status code | yes | / | -| message | String | Information of the response | no | / | -| data | Map | The returned data | yes | / | -| jobExecutionId | String | Task list. The execution status of the Job must be Running before you can get the task list, otherwise the returned task list is empty. Please note: task has no Partial_Success status | yes | / | - -Back to example ： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/tasklist", - "status": 0, - "message": "Submitted succeed(Submit successfully)！", - "data": { - "tasks": [ - { - "taskId": 5, - "name": "test-1", - "status": "Inited", // There is no task Partial_Success status. - "createTime": "2022-01-03 09:00:00", - "launchTime": null, - "lastUpdateTime": "2022-01-03 09:00:00", - "engineType": "sqoop", - "linkisJobId": null, - "linkisJobInfo": null, - "executeUser": "enjoyyin" - } - ] - } -} - -``` - -### 4、Get the execution progress of Job & task - -Interface description： Get the execution progress through jobExecutionId - -Prerequisites: the execution status of the Job must be Running before you can get the progress of the task list. - -Request URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/progress - -Request mode：GET - -Request parameters： - -| Name | Type | Remark | If required | Default value | -| -------------- | ------ | ---------------------------- | ----------- | ------------- | -| jobExecutionId | String | Execution ID of ExchangisJob | yes | / | - -Back to example ： - -| Name | Type | Remark | If required | Default value | -| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | -| method | String | Called method (request path) | yes | / | -| status | int | Response status code | yes | / | -| message | String | Information of the response | no | / | -| data | Map | The returned data | yes | / | -| jobExecutionId | String | Task list. The execution status of the Job must be Running before you can get the task list, otherwise the returned task list is empty | yes | / | - -Back to example ： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/progress", - "status": 0, - "message": "Submitted succeed(Submit successfully)！", - "data": { - "job": { - "status": "Running", - "progress": 0.1, - "tasks": { - "running": [ - { - "taskId": 5, - "name": "test-1", - "status": "Running", - "progress": 0.1 - } - ], - "Inited": [ - { - "taskId": 5, - "name": "test-1", - "status": "Inited", - "progress": 0.1 - } - ], - "Scheduled": [], - "Success": [], - "Failed": [], // If there is a Failed task, the Job will fail directly. - "WaitForRetry": [], - "Cancelled": [], // If there is a Cancelled task, the Job will fail directly - "Undefined": [], // If there is a Undefined task, the Job will fail directly - "Timeout": [] - } - } - } -} - -``` - -### 5、Get the indicator information of task runtime - -Interface description：Through jobExecutionId and taskId, we can get the information of various indicators when task is running. - -Prerequisites: before you can get the indicator information of the task, the execution status of the task must be Running; otherwise, the returned information is empty. - -Request URL：/api/rest_j/v1/exchangis/task/execution/{taskId}/metrics - -Request mode：POST - -Request parameters： - -| Name | Type | Remark | If required | Default value | -| -------------- | ------ | --------------------------------------------------- | ----------- | ------------- | -| jobExecutionId | String | Execution ID of ExchangisJob，put it in requestBody | yes | / | -| taskId | String | Execution ID of task，put it in URI | yes | / | - -Back to example ： - -| Name | Type | Remark | If required | Default value | -| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | -| method | String | Called method (request path) | yes | / | -| status | int | Response status code | yes | / | -| message | String | Information of the response | no | / | -| data | Map | The returned data | yes | / | -| jobExecutionId | String | Information of each task index. The execution status of task must be Running before you can get the indicator information of task | yes | / | - -Back to example ： - -```json -{ - "method": "/api/rest_j/v1/exchangis/task/execution/{taskId}/metrics", - "status": 0, - "message": "Submitted succeed(Submit successfully)！", - "data": { - "task": { - "taskId": 5, - "name": "test-1", - "status": "running", - "metrics": { - "resourceUsed": { - "cpu": 10, // Unit：vcores - "memory": 20 // Unit：GB - }, - "traffic": { - "source": "mysql", - "sink": "hive", - "flow": 100 // Unit：Records/S - }, - "indicator": { - "exchangedRecords": 109345, // Unit：Records - "errorRecords": 5, - "ignoredRecords": 5 - } - } - } - } -} -``` - -### 6、Get the real-time log of Job - -Interface description：Get the real-time log of Job through jobExecutionId. - -Request URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/log? fromLine=&pageSize=&ignoreKeywords=&onlyKeywords=&lastRows= - -Request mode：GET - -Request parameters： - -| Name | Type | Remark | If required | Default value | -| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | -| jobExecutionId | String | Execution ID of ExchangisJob | yes | / | -| fromLine | int | Read the starting line of | no | 0 | -| pageSize | int | Read the number of log lines this time | no | 100 | -| ignoreKeywords | String | Ignore which keywords are in the line, and multiple keywords are separated by English | no | / | -| onlyKeywords | String | Only the lines where keywords are located are selected, and multiple keywords are separated in English | no | / | -| lastRows | int | Read only the last few lines of the log, which is equivalent to tail -f log. When this parameter is greater than 0, all the above parameters will be invalid | no | / | - -Back to example ： - -| Name | Type | Remark | If required | Default value | -| ------- | ------- | ------------------------------------------------------------ | ----------- | ------------- | -| method | String | Called method (request path) | yes | / | -| status | int | Response status code | yes | / | -| message | String | Information of the response | no | / | -| data | Map | The returned data | yes | / | -| endLine | int | The end line number of this reading, you can continue reading the log from endLine+1 next time | yes | / | -| isEnd | boolean | Have all the logs been read | no | / | -| logs | List | Returns the execution log of the Job | yes | / | - -Back to example ： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/log", - "status": 0, - "message": "Submitted succeed(Submit successfully)！", - "data": { - "endLine": 99, // The end line number of this reading, you can continue reading the log from endLine+1 next time - "isEnd": false, // Have all the logs been read - "logs": [ - "all": "", - "error": "", - "warn": "", - "info": "" - ] - } -} -``` - -### 7、Get the real-time log of task - -Interface description： Get the real-time log of task through jobExecutionId and taskId. - -Request URL：/api/rest_j/v1/exchangis/task/execution/{taskId}/log? jobExecutionId=&fromLine=&pageSize=&ignoreKeywords=&onlyKeywords=&lastRows= - -Request mode：GET - -Request parameters： - -| Name | Type | Remark | If required | Default value | -| -------------- | ------ | ------------------------------------------------------------ | ----------- | ------------- | -| taskId | String | Execution ID of task | yes | / | -| jobExecutionId | String | Execution ID of ExchangisJob | yes | / | -| fromLine | int | Read the starting line of | no | 0 | -| pageSize | int | Read the number of log lines this time | no | 100 | -| ignoreKeywords | String | Ignore which keywords are in the line, and multiple keywords are separated by English | no | / | -| onlyKeywords | String | Only the lines where keywords are located are selected, and multiple keywords are separated in English | no | / | -| lastRows | int | Read only the last few lines of the log, which is equivalent to tail -f log. When this parameter is greater than 0, all the above parameters will be invalid | no | / | - -Back to example ： - -| Name | Type | Remark | If required | Default value | -| ------- | ------- | ------------------------------------------------------------ | ----------- | ------------- | -| method | String | Called method (request path) | yes | / | -| status | int | Response status code | yes | / | -| message | String | Information of the response | no | / | -| data | Map | The returned data | yes | / | -| endLine | int | The end line number of this reading, you can continue reading the log from endLine+1 next time. | yes | / | -| isEnd | boolean | Have all the logs been read | yes | / | -| logs | List | Returns the execution log of the Job | yes | / | - -Back to example ： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/execution/{taskId}/log", - "status": 0, - "message": "Submitted succeed(Submit successfully)！", - "data": { - "endLine": 99, // The end line number of this reading, you can continue reading the log from endLine+1 next time. - "isEnd": false, // Have all the logs been read - "logs": [ - "all": "", - "error": "", - "warn": "", - "info": "" - ] - } -} -``` - diff --git a/docs/en_US/ch1/exchangis_sqoop_deploy_en.md b/docs/en_US/ch1/exchangis_sqoop_deploy_en.md index 30f88b4ca..0a64db01a 100644 --- a/docs/en_US/ch1/exchangis_sqoop_deploy_en.md +++ b/docs/en_US/ch1/exchangis_sqoop_deploy_en.md @@ -2,7 +2,7 @@ ### Prepare the environment Sqoop engine is an indispensable component to perform Exchange IS data synchronization task, and only after the installation and deployment of Sqoop engine can it successfully perform data synchronization task. At the same time, make sure sqoop is installed on the deployed machine. -Before you install and deploy Sqoop engine, please follow [Exchangis1.0.1](https://github.com/wedisphere/exchange/blob/dev-1.0.1-rc/docs/zh _ cn/ch1/exchange _ deploy _ cn.md). +Before you install and deploy Sqoop engine, Please complete the installation of Exchangis 1.0.0 and related components according to the [Exchangis 1.0.0 installation and deployment document](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/en_US/ch1/exchangis_deploy_en.md), and ensure that the basic functions of the project are available. Sqoop engine mainly depends on Hadoop basic environment. If this node needs to deploy Sqoop engine, it needs to deploy Hadoop client environment. @@ -28,7 +28,7 @@ It is strongly recommended that you use the native Sqoop to perform the test tas Exchangis1.0.1 and Linkis 1.1.0 support the mainstream Sqoop versions 1.4.6 and 1.4.7, and later versions may need to modify some codes for recompilation. -[Click to jump to Release interface](https://github.com/WeBankFinTech/Exchangis/releases) +[Click to jump to Release interface](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.0.0) #### 2）Compile and package If you want to develop and compile sqoop engine yourself, the specific compilation steps are as follows: diff --git a/docs/en_US/ch1/exchangis_user_manual_en.md b/docs/en_US/ch1/exchangis_user_manual_en.md index 49c22be9d..7ad3b9e6a 100644 --- a/docs/en_US/ch1/exchangis_user_manual_en.md +++ b/docs/en_US/ch1/exchangis_user_manual_en.md @@ -34,7 +34,7 @@ The main functions of data source are as follows ： ![datasource_list](../../../images/zh_CN/ch1/datasource_list.png)
-Pic4-1 Datasource management list +Pic3-1 Datasource management list
@@ -45,7 +45,7 @@ Pic4-1 Datasource management list ![datasource_type](../../../images/zh_CN/ch1/datasource_type.png)
-Pic4-2 Datasource type +Pic3-2 Datasource type
@@ -54,7 +54,7 @@ Pic4-2 Datasource type ![MySQL_datasource_config](../../../images/zh_CN/ch1/MySQL_datasource_config.png)
-Pic4-3 MySQL datasource config +Pic3-3 MySQL datasource config
@@ -63,7 +63,7 @@ Pic4-3 MySQL datasource config ![Hive_datasource_config](../../../images/zh_CN/ch1/Hive_datasource_config.png)
-Pic4-4 Hive datasource config +Pic3-4 Hive datasource config
@@ -74,14 +74,14 @@ Pic4-4 Hive datasource config ![datasource_func](../../../images/zh_CN/ch1/datasource_func.png)
-Pic4-5 Datasource release +Pic3-5 Datasource release
  The **expired ** function of data source management is used to indicate that this data source has been gradually replaced. Please change the task configuration using this data source in time to avoid the failure of the configured execution task caused by directly deleting the data source. ![datasource_timelimit](../../../images/zh_CN/ch1/datasource_timelimit.png)
-Pic4-6 Datasource expiration +Pic3-6 Datasource expiration
## 四、Project management @@ -92,29 +92,47 @@ Pic4-6 Datasource expiration ![item_list](../../../images/zh_CN/ch1/item_list.png)
-Pic3-1 Project list +Pic4-1 Project list
### 2、Task list -   You can manage the created Job data synchronization tasks in the task list, similar to projects, including **Create**, **Modify**, **Delete** and **Search**. +  Enter the project, and you can see the task list under the project. + +#### 1）Task manage + +  You can manage the created Job data synchronization tasks in the task list, similar to projects, including **Create**, **Modify**, **Delete** and **Search**. ![job_task_list](../../../images/zh_CN/ch1/job_task_list.png)
-Pic3-2 Task list +Pic4-2 Task list
-   In addition, the **task supports copying**, which can increase the required tasks. The copied task contains all the information of its original task configuration. Click **Create Task** to select the task type and execution engine. **Currently, only offline tasks and SQOOP execution engine** are supported, and streaming tasks and DataX engines will be supported in the future. +   Click **Create Task** to select the task type and execution engine. **Currently, only offline tasks and SQOOP execution engine** are supported, and streaming tasks and DataX engines will be supported in the future. ![task_type_and_engine](../../../images/zh_CN/ch1/task_type_and_engine.png)
-Pic3-3 Task type and Engine config +Pic4-3 Task type and Engine config
-### 3、Synchronize data task configuration and execution +#### 2）Subtasks manage + +  Click **Add subtask** in the task to add multiple subtasks. You can **modify, copy and delete subtasks**. + +   At the same time, **tasks support copying**, and copied subtasks contain all information of their atomic task configuration. + +![1656571126825](../../../images/zh_CN/ch1/sub_task_manage.png) + +
+Pic4-4 Subtask manage +
-   Data synchronization task configuration and execution is the core function of Exchangis1.0. The basic configuration data synchronization process is: **Add subtasks-> Select source data source database table and destination data source database table (add data source in data source management module for selection)-> Field mapping (default)-> Process control (default)-> Configuration (default)-> Execute**. +#### 3）Synchronize data task configuration and execution + +  Before this step, you need to add data sources in the **data source management module** for selection and **publish**. The current Exchangis version only supports **MySQL data sources and Hive data sources **. + +  Data synchronization task configuration and execution is the core function of Exchangis1.0.0. The basic configuration data synchronization process is: **Add subtask-> Select Source data source and Sink data source-> Field mapping configuration-> Process control-> Task configuration-> Save-> Execute**. The main functions of task execution include: 1. Add, copy and delete subtask cards ； @@ -127,28 +145,46 @@ The main functions of task execution include: 8. Task execution history status view ； 9. Execute the task kill operation. -### 4、Selecting and configuring data sources +##### Selecting and configuring data sources + +  For the newly created data synchronization subtask, you should select the data source library table, and the selected data source is the published data source in the **data source module**. Select the data source to support search, and the search method is to search the library first, and then search the table. -  Click **Add Sub-task** to start creating a data synchronization task. For the newly created data synchronization sub-task, you must first select the data source library table. The data source must be configured in advance in the **data source management module** before it will appear in the task configuration. Select the data source to support search, and the search method is to search the library first, and then search the table. +  When MySQL is the destination data source, it supports **insert** and **update** two writing methods; When it is the source data source, it supports **WHERE conditional statement query**. + +  When Hive is the destination data source, partition information configuration is supported, and the writing methods are **append data** and **overwrite**; When the source data source is **partition information configuration is supported**. -![add_subtask](../../../images/zh_CN/ch1/add_subtask.png) +![add_subtask](../../../images/zh_CN/ch1/data_source_select.png)
-Pic3-4 Add sub-task +Pic4-5 Select datasource
-  When MySQL is the destination data source, it supports **insert** and **update** two writing methods; When it is the source data source, it supports **WHERE conditional statement query**. +![1656574588669](../../../images/zh_CN/ch1/data_source_insert_way.png) -   When Hive is the destination data source, partition information configuration is supported, and the writing methods are **append data** and **overwrite**; When the source data source is **partition information configuration is supported**. +
+Pic4-6 Write mode configuration +
-### 5、Datasource field mapping +##### Datasource field mapping -   When the data source library table information is configured, Exchangis1.0 will automatically generate automatic field mapping between the original data source and the destination data source in the **field mapping** line, and you can choose the field you want to map; When HIVE is the destination data source, its mapping field cannot be modified, which should be noted. +  When the data Source library table information is configured, Exchangis1.0.0 will automatically map the fields of source data source and Sink data source, and you can choose the fields you want to map by yourself, or check whether our fields match. When Hive is a Sink data source, its mapping field cannot be modified. -### 6、Process control +![1656574253553](../../../images/zh_CN/ch1/data_source_field_mapping.png) -   Task execution provides the configuration of the maximum parallel number of tasks and the maximum memory configuration of jobs, which can be changed according to actual needs. +
+Pic4-7 Field mapping +
-### 7、Job execution +##### Process control + +  Task execution provides **configuration of maximum number of parallel jobs** (default is 1) and **configuration of maximum job memory** (default is 1024Mb), which can be changed according to actual needs. + +![1656574125954](../../../images/zh_CN/ch1/task_proccess_control.png) + +
+Pic4-8 Process control +
+ +#### 4）Job execution   Exchangis1.0 support the simultaneous execution of multiple subtasks. After the task configuration is completed, click Execute to start the data synchronization task, and the workbench will pop up at the bottom of the interface. The workbench mainly contains three functions: **running status, real-time log and execution history** .    **Running Status** :You can view the overall progress of the current data synchronization task, including the number of successes and failures of the task, and click on the name of the task to display the information of various running indicators of each task. @@ -262,3 +298,7 @@ Note: For data synchronization tasks performed in sqoop node of DSS, relevant in ![production_center](../../../images/zh_CN/ch1/production_center.png) +
+Pic6-8 Production center +
+ diff --git a/docs/zh_CN/ch1/component_upgrade_cn.md b/docs/zh_CN/ch1/component_upgrade_cn.md index 54908dd0d..fd4501f35 100644 --- a/docs/zh_CN/ch1/component_upgrade_cn.md +++ b/docs/zh_CN/ch1/component_upgrade_cn.md @@ -64,7 +64,28 @@ cd {DSS_INSTALL_HOME}/dss/bin sh ./appconn-refresh.sh ``` -#### 4.2）验证exchangis-appconn是否生效 +#### 4.2）更新Exchangis安装目录下的lib包 + +将从下载链接中得到的exchangis-project-server-1.0.0.jar和exchangis-server-1.0.0.jar两个包放入以下Exchangis安装目录的文件路径下(先删除原有旧的这两个包)： + +```$xslt +lib/exchangis-server +``` + +再通过以下命令完成 Exchangis Server 的更新重启： + + ```shell script + ./sbin/daemon.sh restart server + ``` +# 4.3）更新exchangis-server.propertis文件 + +将exchangis-server.propertis文件中的最后一行进行替换，替换内容如下 + +```$xslt +wds.linkis-session.ticket.key=bdp-user-ticket-id +``` + +#### 4.3）验证exchangis-appconn是否生效 在安装部署完成exchangis-appconn之后，可通过以下步骤初步验证exchangis-appconn是否安装成功。 1. 在DSS工作空间创建一个新的项目 ![image](https://user-images.githubusercontent.com/27387830/169782142-b2fc2633-e605-4553-9433-67756135a6f1.png) @@ -72,4 +93,5 @@ sh ./appconn-refresh.sh 2. 在exchangis端查看是否同步创建项目，创建成功说明appconn安装成功 ![image](https://user-images.githubusercontent.com/27387830/169782337-678f2df0-080a-495a-b59f-a98c5a427cf8.png) -更多使用操作可参照[Exchangis1.0用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_user_manual_cn.md) + + 更多使用操作可参照[Exchangis1.0用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_user_manual_cn.md) diff --git a/docs/zh_CN/ch1/exchangis_deploy_cn.md b/docs/zh_CN/ch1/exchangis_deploy_cn.md index 906d2a523..f0231e268 100644 --- a/docs/zh_CN/ch1/exchangis_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_deploy_cn.md @@ -24,11 +24,11 @@ Exchangis 的安装，主要分为以下四步： 底层依赖组件检查 -$\color{#FF0000}{注意：一定要重新安装dss1.0.1，并且linkis版本必须大于1.1.1，请重新编译linkis，请使用6月15号发布的包}$。 +$\color{#FF0000}{注意：一定要使用最新版的dss1.1.0，及linkis1.1.1}$。 [linkis1.1.1代码地址](https://github.com/apache/incubator-linkis/tree/release-1.1.1) -[DSS1.0.1代码地址](https://github.com/WeBankFinTech/DataSphereStudio/tree/master) +[DSS1.0.1代码地址](https://github.com/WeBankFinTech/DataSphereStudio/tree/dev-1.1.0) datasource启用 @@ -40,7 +40,7 @@ linkis的启动脚本中默认不会启动数据源相关的服务两个服务 #### 1.3 在linkis中配置授权认证 -###### 1）为exchangis加专用token +###### 1）为exchangis加数据源认证的token 通过在linkis数据库中执行以下语句，为Exchangis分配专属token： @@ -70,7 +70,7 @@ INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_t #### 2.1.1下载二进制包 -从 Exchangis 已发布的 release 中 [点击下载exchangis安装包](https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeDatasphere/Exchangis/exchangis1.0.0/wedatasphere-exchangis-1.0.0.tar.gz)，下载最新的安装包。 +从 Exchangis 已发布的 release 中 [点击下载exchangis安装包](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.0.0)，下载最新的安装包。 #### 2.1.2 编译打包 @@ -188,7 +188,7 @@ DATABASE={dbName} #### 2.7.1 获取前端安装包 -Exchangis 已默认提供了编译好的前端安装包，可直接下载使用：[点击下载前端安装包](https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeDatasphere/Exchangis/exchangis1.0.0/dist.zip) +Exchangis 已默认提供了编译好的前端安装包，可直接下载使用：[点击下载前端安装包](https://github.com/WeBankFinTech/Exchangis/releases/download/release-1.0.0/web-dist.zip) 您也可以自行编译 Exchangis 前端，在 Exchangis 根目录下执行如下命令： @@ -271,7 +271,7 @@ Exchangis 已默认提供了编译好的前端安装包，可直接下载使用 ## 3. DSS ExchangisAppConn 安装部署 -如您想正常使用 Exchangis1.0.0 前端，还需安装 DSS ExchangisAppConn 插件，请参考: [ExchangisAppConn 插件安装文档](exchangis_appconn_deploy_cn.md) +如您想正常使用 Exchangis1.0.0 前端，还需安装 DSS ExchangisAppConn 插件，请参考: [ExchangisAppConn 插件安装文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) ## 4. Linkis Sqoop 引擎安装部署 diff --git a/docs/zh_CN/ch1/exchangis_interface_cn.md b/docs/zh_CN/ch1/exchangis_interface_cn.md new file mode 100644 index 000000000..bf50d1f5f --- /dev/null +++ b/docs/zh_CN/ch1/exchangis_interface_cn.md @@ -0,0 +1,839 @@ +# Exchangis接口文档 + +## Exchangis数据源模块 + +### 1、获取数据源类型 + +接口描述：根据request的信息来获取数据源类型 + +请求URL：/dss/exchangis/main/datasources/type + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 备注 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | ---- | -------- | ------ | ----------- | +| request | HttpServletRequest | | 是 | / | request请求 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------ | -------- | ------ | -------------------- | +| method | String | 是 | | 调用的方法(请求路径) | +| status | int | 是 | | 响应状态码 | +| message | String | 否 | | 响应的信息 | +| data | List | 是 | | 返回的数据 | + +### 2、查询数据源 + +接口描述：根据vo查询所需的数据源 + +请求URL：/dss/exchangis/main/datasources/query + +请求方式：GET、POST + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| vo | DataSourceQueryVO | 是 | / | | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------ | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | List | 是 | / | 返回的数据 | + +### 3、查询数据源 + +接口描述：根据request信息查询数据源 + +请求URL：/dss/exchangis/main/datasources + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| -------- | ------------------ | -------- | ------ | ------------ | +| request | HttpServletRequest | 是 | / | request请求 | +| typeId | Long | 是 | / | 数据源类型ID | +| typeName | String | 是 | / | 数据源类型 | +| page | Integer | 是 | / | 页数 | +| size | Integer | 是 | / | 每页大小 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | --------------------- | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | List\ | 是 | / | 返回的数据 | + +### 4、查询数据源关键定义 + +接口描述：根据数据源类型ID查询数据源关键定义 + +请求URL：/dss/exchangis/main/datasources/types/{dataSourceTypeId}/keydefines + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ---------------- | ------------------ | -------- | ------ | ------------ | +| request | HttpServletRequest | 是 | / | request请求 | +| dataSourceTypeId | Long | 是 | / | 数据源类型ID | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ---------------------- | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | List[Map[String, Any]] | 是 | / | 返回的数据 | + +### 5、获取数据源版本 + +接口描述：根据数据源ID获取数据源版本 + +请求URL：/dss/exchangis/main/datasources/{id}/versions + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| id | Long | 是 | / | 数据源ID | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------------- | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | List> | 是 | / | 返回的数据 | + +### 6、创建数据源 + +接口描述：根据数据源创建VO创建数据源 + +请求URL：/dss/exchangis/main/datasources + +请求方式：POST + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------------------ | ------------------ | -------- | ------ | ------------ | +| request | HttpServletRequest | 是 | / | request请求 | +| dataSourceCreateVO | DataSourceCreateVO | 是 | / | 数据源创建VO | +| bindingResult | BindingResult | 是 | / | 绑定结果 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------ | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | Long | 是 | / | 返回的数据 | + +### 7、获取数据源信息 + +接口描述：根据request信息、数据源ID和版本ID获取数据源信息 + +请求URL：/dss/exchangis/main/datasources/{id} + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| --------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| id | Long | 是 | / | 数据源ID | +| versionId | String | 是 | / | 版本ID | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ----------------- | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | DataSourceItemDTO | 是 | / | 返回的数据 | + +### 8、获取数据源连接参数 + +接口描述：根据request信息和数据源ID获取数据源连接参数 + +请求URL：/dss/exchangis/main/datasources/{id}/connect_params + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| Long | id | 是 | / | 数据源ID | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | -------------------------- | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | java.util.Map[String, Any] | 是 | / | 返回的数据 | + +### 9、更新数据源 + +接口描述：根据request信息、数据源类型ID和数据源创建VO更新数据源 + +请求URL：/dss/exchangis/main/datasources/{id} + +请求方式：PUT + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------------------ | ------------------ | -------- | ------ | ------------ | +| request | HttpServletRequest | 是 | / | request请求 | +| id | Long | 是 | / | 数据源类型ID | +| dataSourceCreateVO | DataSourceCreateVO | 是 | / | 数据源创建VO | +| bindingResult | BindingResult | 是 | / | 绑定结果 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------ | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | Map | 是 | / | 返回的数据 | + +### 10、发布数据源 + +接口描述：根据request请求、数据源ID和版本ID发布数据源 + +请求URL：/dss/exchangis/main/datasources/{id}/{version}/publish + +请求方式：PUT + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| id | Long | 是 | / | 数据源ID | +| version | Long | 是 | / | 版本ID | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------ | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | Map | 是 | / | 返回的数据 | + +### 11、过期数据源 + +接口描述：根据request信息和数据源ID过期数据源 + +请求URL：/dss/exchangis/main/datasources/{id}/expire + +请求方式：PUT + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| id | Long | 是 | / | 数据源ID | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------ | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | Map | 是 | / | 返回的数据 | + +### 12、连接数据源 + +接口描述：根据request信息、数据源ID和版本ID连接数据源 + +请求URL：/dss/exchangis/main/datasources/{id}/{version}/connect + +请求方式：PUT + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| id | Long | 是 | / | 数据源ID | +| version | Long | 是 | / | 版本ID | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------ | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | Map | 是 | / | 返回的数据 | + +### 13、连接数据源 + +接口描述：根据request信息和数据源ID连接数据源 + +请求URL：/dss/exchangis/main/datasources/op/connect + +请求方式：POST + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| id | Long | 是 | / | 数据源ID | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------ | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | Map | 是 | / | 返回的数据 | + +### 14、删除数据源 + +接口描述：根据request信息和数据源类型ID连接数据源 + +请求URL：/dss/exchangis/main/datasources/{id} + +请求方式：DELETE + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ---------------- | ------------------ | -------- | ------ | ------------ | +| request | HttpServletRequest | 是 | / | request请求 | +| dataSourceTypeId | Long | 是 | / | 数据源类型ID | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------ | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | Long | 是 | / | 返回的数据 | + +### 15、根据数据源ID查询数据源 + +接口描述：根据request信息、数据源类型和数据源ID查询数据源 + +请求URL：/dss/exchangis/main/datasources/{type}/{id}/dbs + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| type | String | 是 | / | 数据源类型 | +| id | Long | 是 | / | 数据源ID | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | -------------- | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | List\ | 是 | / | 返回的数据 | + +### 16、根据数据源ID和数据库获取数据表 + +接口描述：根据request信息、数据源类型、数据源ID和数据库获取数据表 + +请求URL：/dss/exchangis/main/datasources/{type}/{id}/dbs/{dbName}/tables + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| type | String | 是 | / | 数据源类型 | +| id | Long | 是 | / | 数据源ID | +| dbName | String | 是 | / | 数据库名 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | -------------- | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | List\ | 是 | / | 返回的数据 | + +### 17、根据数据源ID、数据库和数据表获取表字段 + +接口描述：根据request信息、数据源类型、数据源ID和数据库名和数据表获取表字段 + +请求URL：/dss/exchangis/main/datasources/{type}/{id}/dbs/{dbName}/tables/{tableName}/fields + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| --------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| type | String | 是 | / | 数据源类型 | +| id | Long | 是 | / | 数据源ID | +| dbName | String | 是 | / | 数据库名 | +| tableName | String | 是 | / | 数据表 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ---------------------------------- | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | List\ | 是 | / | 返回的数据 | + +### 18、获取表字段列表信息 + +接口描述：根据request信息和字段映射VO获取表字段列表信息 + +请求URL：/dss/exchangis/main/datasources/fieldsmapping + +请求方式：POST + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | ----------- | +| request | HttpServletRequest | 是 | / | request请求 | +| vo | FieldMappingVO | 是 | / | 字段映射VO | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------------- | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | List> | 是 | / | 返回的数据 | + +### 19、根据数据源类型获取参数 + +接口描述：根据request信息、引擎、数据源类型和文件系统路径获取参数 + +请求URL：/dss/exchangis/main/datasources/{engine}/{type}/params/ui + +请求方式：POST + +请求参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | ------------ | +| request | HttpServletRequest | 是 | / | request请求 | +| engine | String | 是 | / | 引擎 | +| type | String | 是 | / | 数据源类型 | +| dir | String | 是 | / | 文件系统路径 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------------------ | -------- | ------ | -------------------- | +| method | String | 是 | / | 调用的方法(请求路径) | +| status | int | 是 | / | 响应状态码 | +| message | String | 否 | / | 响应的信息 | +| data | List> | 是 | / | 返回的数据 | + + + +## Exchangis作业执行模块 + +### 1、提交配置好的job进行执行 + +接口描述：提交执行ExchangisJob，后台返回jobExecutionId + +请求URL：/api/rest_j/v1/exchangis/job/{id}/execute + +请求方式：POST + +请求参数： + +| 名称 | 类型 | 备注 | 是否必须 | 默认值 | +| --------------------- | ------- | ------------------------------------------------------------ | -------- | ------ | +| id | Long | Exchangis的ID | 是 | 无 | +| permitPartialFailures | boolean | 是否允许部分失败。如果为true，就算 部分子任务失败，整个Job还是会继续 执行，执行完成后，Job状态为 Partial_Success。该参数为 requestBody参数。 | 否 | false | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| -------------- | ------ | -------- | ------ | -------------------- | +| method | String | 是 | | 调用的方法(请求路径) | +| status | int | 是 | | 响应状态码 | +| message | String | 否 | | 响应的信息 | +| data | Map | 是 | | 返回的数据 | +| jobExecutionId | String | 是 | | 执行job的执行id | + +返回示例： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/{id}/execute", + "status": 0, + "message": "Submitted succeed(提交成功)！", + "data": { + "jobExecutionId": "555node1node2node3execId1" +} + +``` + +### 2、获取Job的执行状态 + +接口描述：根据jobExecutionId获取Job的状态 + +请求URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/status + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 备注 | 是否必须 | 默认值 | +| -------------- | ------ | -------------------- | -------- | ------ | +| jobExecutionId | String | ExchangisJob的执行ID | 是 | 无 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| -------------- | ------ | -------- | ------ | ------------------------------------------------------------ | +| method | String | 是 | | 调用的方法(请求路径) | +| status | int | 是 | | 响应状态码 | +| message | String | 否 | | 响应的信息 | +| data | Map | 是 | | 返回的数据 | +| jobExecutionId | String | 是 | | 执行job的状态，包含：Inited，Scheduled， Running，WaitForRetry，Cancelled，Failed， Partial_Success，Success，Undefined，Timeout。其 中，Running状态表示正在运行，从Cancelled开始，都 是已完成状态。 | + +返回示例： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/execution/{id}/status", + "status": 0, + "message": "Submitted succeed(提交成功)！", + "data": { + "status": "Running", + "progress": 0.1 +} +``` + +### 3、获取本次Job执行的task列表 + +接口描述：通过jobExecutionId，获取任务列表 + +前提条件：Job的执行状态必须为Running后，才可以拿到task列表，否则返回的task列表为空 + +请求URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/tasklist + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 备注 | 是否必须 | 默认值 | +| -------------- | ------ | ------ | -------- | ------ | +| jobExecutionId | String | string | 是 | 无 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| -------------- | ------ | -------- | ------ | ------------------------------------------------------------ | +| method | String | 是 | | 调用的方法(请求路径) | +| status | int | 是 | | 响应状态码 | +| message | String | 否 | | 响应的信息 | +| data | Map | 是 | | 返回的数据 | +| jobExecutionId | String | 是 | | 任务列表。Job的执行状态必须为Running后，才可以 拿到task列表，否则返回的task列表为空。 请注意：task没有Partial_Success状态。 | + +返回示例： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/tasklist", + "status": 0, + "message": "Submitted succeed(提交成功)！", + "data": { + "tasks": [ + { + "taskId": 5, + "name": "test-1", + "status": "Inited", // task没有Partial_Success状态 + "createTime": "2022-01-03 09:00:00", + "launchTime": null, + "lastUpdateTime": "2022-01-03 09:00:00", + "engineType": "sqoop", + "linkisJobId": null, + "linkisJobInfo": null, + "executeUser": "enjoyyin" + } + ] + } +} + +``` + +### 4、获取Job & task的执行进度 + +接口描述：通过jobExecutionId，获取执行进度 + +前提条件：Job的执行状态必须为Running后，才可以拿到task列表的进度 + +请求URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/progress + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 备注 | 是否必须 | 默认值 | +| -------------- | ------ | -------------------- | -------- | ------ | +| jobExecutionId | String | ExchangisJob的执行ID | 是 | 无 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| -------------- | ------ | -------- | ------ | ------------------------------------------------------------ | +| method | String | 是 | | 调用的方法(请求路径) | +| status | int | 是 | | 响应状态码 | +| message | String | 否 | | 响应的信息 | +| data | Map | 是 | | 返回的数据 | +| jobExecutionId | String | 是 | | 任务列表。Job的执行状态必须为Running后，才可以 拿到task列表，否则返回的task列表为空。 | + +返回示例： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/progress", + "status": 0, + "message": "Submitted succeed(提交成功)！", + "data": { + "job": { + "status": "Running", + "progress": 0.1, + "tasks": { + "running": [ + { + "taskId": 5, + "name": "test-1", + "status": "Running", + "progress": 0.1 + } + ], + "Inited": [ + { + "taskId": 5, + "name": "test-1", + "status": "Inited", + "progress": 0.1 + } + ], + "Scheduled": [], + "Success": [], + "Failed": [], // 如果存在Failed的task，则Job会直接失败 + "WaitForRetry": [], + "Cancelled": [], // 如果存在Cancelled的task，则Job会直接失败 + "Undefined": [], // 如果存在Undefined的task，则Job会直接失败 + "Timeout": [] + } + } + } +} + +``` + +### 5、获取task运行时的各项指标信息 + +接口描述：通过jobExecutionId和taskId，获取task运行时的各项指标信息 + +前提条件：task的执行状态必须为Running后，才可以拿到task的各项指标信息，否则返回的 为空 + +请求URL：/api/rest_j/v1/exchangis/task/execution/{taskId}/metrics + +请求方式：POST + +请求参数： + +| 名称 | 类型 | 备注 | 是否必须 | 默认值 | +| -------------- | ------ | ---------------------------------------- | -------- | ------ | +| jobExecutionId | String | ExchangisJob的执行ID，放入 requestBody中 | 是 | 无 | +| taskId | String | task的执行ID，放入URI中 | 是 | 无 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| -------------- | ------ | -------- | ------ | ------------------------------------------------------------ | +| method | String | 是 | | 调用的方法(请求路径) | +| status | int | 是 | | 响应状态码 | +| message | String | 否 | | 响应的信息 | +| data | Map | 是 | | 返回的数据 | +| jobExecutionId | String | 是 | | 任务各项指标信息。task的执行状态必须为 Running后，才可以拿到task的各项指标信息。 | + +返回示例： + +```json +{ + "method": "/api/rest_j/v1/exchangis/task/execution/{taskId}/metrics", + "status": 0, + "message": "Submitted succeed(提交成功)！", + "data": { + "task": { + "taskId": 5, + "name": "test-1", + "status": "running", + "metrics": { + "resourceUsed": { + "cpu": 10, // 单位：vcores + "memory": 20 // 单位：GB + }, + "traffic": { + "source": "mysql", + "sink": "hive", + "flow": 100 // 单位：Records/S + }, + "indicator": { + "exchangedRecords": 109345, // 单位：Records + "errorRecords": 5, + "ignoredRecords": 5 + } + } + } + } +} +``` + +### 6、获取Job的实时日志 + +接口描述：通过jobExecutionId，获取Job的实时日志 + +请求URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/log? fromLine=&pageSize=&ignoreKeywords=&onlyKeywords=&lastRows= + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 备注 | 是否必须 | 默认值 | +| -------------- | ------ | ------------------------------------------------------------ | -------- | ------ | +| jobExecutionId | String | ExchangisJob的执行ID | 是 | 无 | +| fromLine | int | 读取的起始行 | 否 | 0 | +| pageSize | int | 本次读取日志行数 | 否 | 100 | +| ignoreKeywords | String | 忽略哪些关键字所在的行，多个关键字以英 文,分隔 | 否 | 无 | +| onlyKeywords | String | 只取哪些关键字所在的行，多个关键字以英 文,分隔 | 否 | 无 | +| lastRows | int | 只读取最后多少行的日志，相当于tail -f log。该参数大于0时，上面所有的参数都会 失效。 | 否 | 无 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------- | -------- | ------ | ------------------------------------------------------ | +| method | String | 是 | | 调用的方法(请求路径) | +| status | int | 是 | | 响应状态码 | +| message | String | 否 | | 响应的信息 | +| data | Map | 是 | | 返回的数据 | +| endLine | int | 是 | | 本次读取的结束行号,下次可以从 endLine + 1 继续读取日志 | +| isEnd | boolean | | | 日志是否已经全部读完 | +| logs | List | 是 | | 返回Job的执行日志 | + +返回示例： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/log", + "status": 0, + "message": "Submitted succeed(提交成功)！", + "data": { + "endLine": 99, // 本次读取的结束行号,下次可以从 endLine + 1 继续读取日志 + "isEnd": false, // 日志是否已经全部读完 + "logs": [ + "all": "", + "error": "", + "warn": "", + "info": "" + ] + } +} +``` + +### 7、获取task的实时日志 + +接口描述：通过jobExecutionId和taskId，获取task的实时日志 + +请求URL：/api/rest_j/v1/exchangis/task/execution/{taskId}/log? jobExecutionId=&fromLine=&pageSize=&ignoreKeywords=&onlyKeywords=&lastRows= + +请求方式：GET + +请求参数： + +| 名称 | 类型 | 备注 | 是否必须 | 默认值 | +| -------------- | ------ | ------------------------------------------------------------ | -------- | ------ | +| taskId | String | task的执行Id | 是 | 无 | +| jobExecutionId | String | ExchangisJob的执行ID | 是 | 无 | +| fromLine | int | 读取的起始行 | 否 | 0 | +| pageSize | int | 本次读取日志行数 | 否 | 100 | +| ignoreKeywords | String | 忽略哪些关键字所在的行，多个关键字以英 文,分隔 | 否 | 无 | +| onlyKeywords | String | 只取哪些关键字所在的行，多个关键字以英 文,分隔 | 否 | 无 | +| lastRows | int | 只读取最后多少行的日志，相当于tail -f log。该参数大于0时，上面所有的参数都会 失效。 | 否 | 无 | + +返回参数： + +| 名称 | 类型 | 是否必须 | 默认值 | 备注 | +| ------- | ------- | -------- | ------ | ------------------------------------------------------ | +| method | String | 是 | | 调用的方法(请求路径) | +| status | int | 是 | | 响应状态码 | +| message | String | 否 | | 响应的信息 | +| data | Map | 是 | | 返回的数据 | +| endLine | int | 是 | | 本次读取的结束行号,下次可以从 endLine + 1 继续读取日志 | +| isEnd | boolean | | | 日志是否已经全部读完 | +| logs | List | 是 | | 返回Job的执行日志 | + +返回示例： + +```json +{ + "method": "/api/rest_j/v1/exchangis/job/execution/{taskId}/log", + "status": 0, + "message": "Submitted succeed(提交成功)！", + "data": { + "endLine": 99, // 本次读取的结束行号,下次可以从 endLine + 1 继续读取日志 + "isEnd": false, // 日志是否已经全部读完 + "logs": [ + "all": "", + "error": "", + "warn": "", + "info": "" + ] + } +} +``` + diff --git a/docs/zh_CN/ch1/exchangis_job_execute_interface_cn.md b/docs/zh_CN/ch1/exchangis_job_execute_interface_cn.md deleted file mode 100644 index 9c71a68fc..000000000 --- a/docs/zh_CN/ch1/exchangis_job_execute_interface_cn.md +++ /dev/null @@ -1,362 +0,0 @@ -# Exchangis作业执行模块接口文档 - -### 1、提交配置好的job进行执行 - -接口描述：提交执行ExchangisJob，后台返回jobExecutionId - -请求URL：/api/rest_j/v1/exchangis/job/{id}/execute - -请求方式：POST - -请求参数： - -| 名称 | 类型 | 备注 | 是否必须 | 默认值 | -| --------------------- | ------- | ------------------------------------------------------------ | -------- | ------ | -| id | Long | Exchangis的ID | 是 | 无 | -| permitPartialFailures | boolean | 是否允许部分失败。如果为true，就算 部分子任务失败，整个Job还是会继续 执行，执行完成后，Job状态为 Partial_Success。该参数为 requestBody参数。 | 否 | false | - -返回参数： - -| 名称 | 类型 | 是否必须 | 默认值 | 备注 | -| -------------- | ------ | -------- | ------ | -------------------- | -| method | String | 是 | | 调用的方法(请求路径) | -| status | int | 是 | | 响应状态码 | -| message | String | 否 | | 响应的信息 | -| data | Map | 是 | | 返回的数据 | -| jobExecutionId | String | 是 | | 执行job的执行id | - -返回示例： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/{id}/execute", - "status": 0, - "message": "Submitted succeed(提交成功)！", - "data": { - "jobExecutionId": "555node1node2node3execId1" -} - -``` - -### 2、获取Job的执行状态 - -接口描述：根据jobExecutionId获取Job的状态 - -请求URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/status - -请求方式：GET - -请求参数： - -| 名称 | 类型 | 备注 | 是否必须 | 默认值 | -| -------------- | ------ | -------------------- | -------- | ------ | -| jobExecutionId | String | ExchangisJob的执行ID | 是 | 无 | - -返回参数： - -| 名称 | 类型 | 是否必须 | 默认值 | 备注 | -| -------------- | ------ | -------- | ------ | ------------------------------------------------------------ | -| method | String | 是 | | 调用的方法(请求路径) | -| status | int | 是 | | 响应状态码 | -| message | String | 否 | | 响应的信息 | -| data | Map | 是 | | 返回的数据 | -| jobExecutionId | String | 是 | | 执行job的状态，包含：Inited，Scheduled， Running，WaitForRetry，Cancelled，Failed， Partial_Success，Success，Undefined，Timeout。其 中，Running状态表示正在运行，从Cancelled开始，都 是已完成状态。 | - -返回示例： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/execution/{id}/status", - "status": 0, - "message": "Submitted succeed(提交成功)！", - "data": { - "status": "Running", - "progress": 0.1 -} -``` - -### 3、获取本次Job执行的task列表 - -接口描述：通过jobExecutionId，获取任务列表 - -前提条件：Job的执行状态必须为Running后，才可以拿到task列表，否则返回的task列表为空 - -请求URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/tasklist - -请求方式：GET - -请求参数： - -| 名称 | 类型 | 备注 | 是否必须 | 默认值 | -| -------------- | ------ | ------ | -------- | ------ | -| jobExecutionId | String | string | 是 | 无 | - -返回参数： - -| 名称 | 类型 | 是否必须 | 默认值 | 备注 | -| -------------- | ------ | -------- | ------ | ------------------------------------------------------------ | -| method | String | 是 | | 调用的方法(请求路径) | -| status | int | 是 | | 响应状态码 | -| message | String | 否 | | 响应的信息 | -| data | Map | 是 | | 返回的数据 | -| jobExecutionId | String | 是 | | 任务列表。Job的执行状态必须为Running后，才可以 拿到task列表，否则返回的task列表为空。 请注意：task没有Partial_Success状态。 | - -返回示例： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/tasklist", - "status": 0, - "message": "Submitted succeed(提交成功)！", - "data": { - "tasks": [ - { - "taskId": 5, - "name": "test-1", - "status": "Inited", // task没有Partial_Success状态 - "createTime": "2022-01-03 09:00:00", - "launchTime": null, - "lastUpdateTime": "2022-01-03 09:00:00", - "engineType": "sqoop", - "linkisJobId": null, - "linkisJobInfo": null, - "executeUser": "enjoyyin" - } - ] - } -} - -``` - -### 4、获取Job & task的执行进度 - -接口描述：通过jobExecutionId，获取执行进度 - -前提条件：Job的执行状态必须为Running后，才可以拿到task列表的进度 - -请求URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/progress - -请求方式：GET - -请求参数： - -| 名称 | 类型 | 备注 | 是否必须 | 默认值 | -| -------------- | ------ | -------------------- | -------- | ------ | -| jobExecutionId | String | ExchangisJob的执行ID | 是 | 无 | - -返回参数： - -| 名称 | 类型 | 是否必须 | 默认值 | 备注 | -| -------------- | ------ | -------- | ------ | ------------------------------------------------------------ | -| method | String | 是 | | 调用的方法(请求路径) | -| status | int | 是 | | 响应状态码 | -| message | String | 否 | | 响应的信息 | -| data | Map | 是 | | 返回的数据 | -| jobExecutionId | String | 是 | | 任务列表。Job的执行状态必须为Running后，才可以 拿到task列表，否则返回的task列表为空。 | - -返回示例： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/progress", - "status": 0, - "message": "Submitted succeed(提交成功)！", - "data": { - "job": { - "status": "Running", - "progress": 0.1, - "tasks": { - "running": [ - { - "taskId": 5, - "name": "test-1", - "status": "Running", - "progress": 0.1 - } - ], - "Inited": [ - { - "taskId": 5, - "name": "test-1", - "status": "Inited", - "progress": 0.1 - } - ], - "Scheduled": [], - "Success": [], - "Failed": [], // 如果存在Failed的task，则Job会直接失败 - "WaitForRetry": [], - "Cancelled": [], // 如果存在Cancelled的task，则Job会直接失败 - "Undefined": [], // 如果存在Undefined的task，则Job会直接失败 - "Timeout": [] - } - } - } -} - -``` - -### 5、获取task运行时的各项指标信息 - -接口描述：通过jobExecutionId和taskId，获取task运行时的各项指标信息 - -前提条件：task的执行状态必须为Running后，才可以拿到task的各项指标信息，否则返回的 为空 - -请求URL：/api/rest_j/v1/exchangis/task/execution/{taskId}/metrics - -请求方式：POST - -请求参数： - -| 名称 | 类型 | 备注 | 是否必须 | 默认值 | -| -------------- | ------ | ---------------------------------------- | -------- | ------ | -| jobExecutionId | String | ExchangisJob的执行ID，放入 requestBody中 | 是 | 无 | -| taskId | String | task的执行ID，放入URI中 | 是 | 无 | - -返回参数： - -| 名称 | 类型 | 是否必须 | 默认值 | 备注 | -| -------------- | ------ | -------- | ------ | ------------------------------------------------------------ | -| method | String | 是 | | 调用的方法(请求路径) | -| status | int | 是 | | 响应状态码 | -| message | String | 否 | | 响应的信息 | -| data | Map | 是 | | 返回的数据 | -| jobExecutionId | String | 是 | | 任务各项指标信息。task的执行状态必须为 Running后，才可以拿到task的各项指标信息。 | - -返回示例： - -```json -{ - "method": "/api/rest_j/v1/exchangis/task/execution/{taskId}/metrics", - "status": 0, - "message": "Submitted succeed(提交成功)！", - "data": { - "task": { - "taskId": 5, - "name": "test-1", - "status": "running", - "metrics": { - "resourceUsed": { - "cpu": 10, // 单位：vcores - "memory": 20 // 单位：GB - }, - "traffic": { - "source": "mysql", - "sink": "hive", - "flow": 100 // 单位：Records/S - }, - "indicator": { - "exchangedRecords": 109345, // 单位：Records - "errorRecords": 5, - "ignoredRecords": 5 - } - } - } - } -} -``` - -### 6、获取Job的实时日志 - -接口描述：通过jobExecutionId，获取Job的实时日志 - -请求URL：/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/log? fromLine=&pageSize=&ignoreKeywords=&onlyKeywords=&lastRows= - -请求方式：GET - -请求参数： - -| 名称 | 类型 | 备注 | 是否必须 | 默认值 | -| -------------- | ------ | ------------------------------------------------------------ | -------- | ------ | -| jobExecutionId | String | ExchangisJob的执行ID | 是 | 无 | -| fromLine | int | 读取的起始行 | 否 | 0 | -| pageSize | int | 本次读取日志行数 | 否 | 100 | -| ignoreKeywords | String | 忽略哪些关键字所在的行，多个关键字以英 文,分隔 | 否 | 无 | -| onlyKeywords | String | 只取哪些关键字所在的行，多个关键字以英 文,分隔 | 否 | 无 | -| lastRows | int | 只读取最后多少行的日志，相当于tail -f log。该参数大于0时，上面所有的参数都会 失效。 | 否 | 无 | - -返回参数： - -| 名称 | 类型 | 是否必须 | 默认值 | 备注 | -| ------- | ------- | -------- | ------ | ------------------------------------------------------ | -| method | String | 是 | | 调用的方法(请求路径) | -| status | int | 是 | | 响应状态码 | -| message | String | 否 | | 响应的信息 | -| data | Map | 是 | | 返回的数据 | -| endLine | int | 是 | | 本次读取的结束行号,下次可以从 endLine + 1 继续读取日志 | -| isEnd | boolean | | | 日志是否已经全部读完 | -| logs | List | 是 | | 返回Job的执行日志 | - -返回示例： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/log", - "status": 0, - "message": "Submitted succeed(提交成功)！", - "data": { - "endLine": 99, // 本次读取的结束行号,下次可以从 endLine + 1 继续读取日志 - "isEnd": false, // 日志是否已经全部读完 - "logs": [ - "all": "", - "error": "", - "warn": "", - "info": "" - ] - } -} -``` - -### 7、获取task的实时日志 - -接口描述：通过jobExecutionId和taskId，获取task的实时日志 - -请求URL：/api/rest_j/v1/exchangis/task/execution/{taskId}/log? jobExecutionId=&fromLine=&pageSize=&ignoreKeywords=&onlyKeywords=&lastRows= - -请求方式：GET - -请求参数： - -| 名称 | 类型 | 备注 | 是否必须 | 默认值 | -| -------------- | ------ | ------------------------------------------------------------ | -------- | ------ | -| taskId | String | task的执行Id | 是 | 无 | -| jobExecutionId | String | ExchangisJob的执行ID | 是 | 无 | -| fromLine | int | 读取的起始行 | 否 | 0 | -| pageSize | int | 本次读取日志行数 | 否 | 100 | -| ignoreKeywords | String | 忽略哪些关键字所在的行，多个关键字以英 文,分隔 | 否 | 无 | -| onlyKeywords | String | 只取哪些关键字所在的行，多个关键字以英 文,分隔 | 否 | 无 | -| lastRows | int | 只读取最后多少行的日志，相当于tail -f log。该参数大于0时，上面所有的参数都会 失效。 | 否 | 无 | - -返回参数： - -| 名称 | 类型 | 是否必须 | 默认值 | 备注 | -| ------- | ------- | -------- | ------ | ------------------------------------------------------ | -| method | String | 是 | | 调用的方法(请求路径) | -| status | int | 是 | | 响应状态码 | -| message | String | 否 | | 响应的信息 | -| data | Map | 是 | | 返回的数据 | -| endLine | int | 是 | | 本次读取的结束行号,下次可以从 endLine + 1 继续读取日志 | -| isEnd | boolean | | | 日志是否已经全部读完 | -| logs | List | 是 | | 返回Job的执行日志 | - -返回示例： - -```json -{ - "method": "/api/rest_j/v1/exchangis/job/execution/{taskId}/log", - "status": 0, - "message": "Submitted succeed(提交成功)！", - "data": { - "endLine": 99, // 本次读取的结束行号,下次可以从 endLine + 1 继续读取日志 - "isEnd": false, // 日志是否已经全部读完 - "logs": [ - "all": "", - "error": "", - "warn": "", - "info": "" - ] - } -} -``` - -### \ No newline at end of file diff --git a/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md b/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md index 56b90df3d..8640ff7ef 100644 --- a/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md @@ -2,7 +2,7 @@ ### 环境准备 Sqoop引擎是执行Exchangis数据同步任务不可或缺的组件，只有安装部署完成Sqoop引擎才能够成功执行数据同步任务。同时，确保所部署的机器上有安装sqoop。 -您在安装部署Sqoop引擎之前，请按照[Exchangis1.0.0](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_deploy_cn.md)安装部署文档安装完成Exchangis1.0.0及相关组件的安装，并确保工程基本功能可用。 +您在安装部署Sqoop引擎之前，请按照[Exchangis1.0.0安装部署文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_deploy_cn.md)安装完成Exchangis1.0.0及相关组件的安装，并确保工程基本功能可用。 Sqoop引擎主要依赖Hadoop基础环境，如果该节点需要部署Sqoop引擎，需要部署Hadoop客户端环境。 @@ -22,14 +22,13 @@ Sqoop引擎主要依赖Hadoop基础环境，如果该节点需要部署Sqoop引 | Linkis系统参数 | 参数 | 备注 | | --------------------------- | ------------------------------- | ------------------------------------------------------------ | | wds.linkis.hadoop.site.xml | 设置sqoop加载hadoop参数文件位置 | 必须，参考示例："/etc/hadoop/conf/core-site.xml;/etc/hadoop/conf/hdfs-site.xml;/etc/hadoop/conf/yarn-site.xml;/etc/hadoop/conf/mapred-site.xml" | -| sqoop.fetch.status.interval | 设置获取sqoop执行状态的间隔时间 | 非必须，默认值为5s - +| sqoop.fetch.status.interval | 设置获取sqoop执行状态的间隔时间 | 非必须，默认值为5s | ### 安装包准备 #### 1）下载二进制包 Exchangis1.0.0和Linkis 1.1.0支持的主流Sqoop版本1.4.6与1.4.7，更高版本可能需要修改部分代码重新编译。 -[点击跳转 Release 界面](https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeDatasphere/Exchangis/exchangis1.0.1-rc/sqoop.zip) +[点击跳转 Release 界面](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.0.0) #### 2） 编译打包 如果您想自己开发和编译sqoop引擎，具体编译步骤如下： @@ -43,30 +42,31 @@ mvn clean install ``` 然后会在该路径下找到sqoop引擎安装包 ``` -{EXCHANGIS_CODE_HOME}/exchangis-plugins/sqoop/target/out/sqoop +{EXCHANGIS_CODE_HOME}/exchangis-plugins/sqoop/target/out ``` ### 开始部署 #### 1）sqoop引擎安装 -1.拿到打包出来的sqoop.zip物料包 +1.拿到打包出来的sqoop物料包，目录结构为： -2.放置到如下目录并进行解压 +```shell +sqoop +-- dist +-- plugin ``` + +2.放置到linkis安装路径的如下目录 + +```shell cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins -unzip.zip -``` -解压出来的目录结构为： -``` -dist -plugin ``` (注意，看当前sqoop引擎对哪些用户有权限，不一定是root) #### 2）重启linkis-engineplugin服务使sqoop引擎生效 -新加入linkis的引擎都要重启linkis的engineplugin服务才会生效，重启脚本为linkis安装目录下的./linkis-daemon.sh，具体步骤如下 -``` +新加入linkis的引擎都要重启linkis的engineplugin服务才会生效，重启脚本为linkis安装目录下的./sbin/linkis-daemon.sh，具体步骤如下 +```shell cd {LINKIS_INSTALL_HOME}/links/sbin/ ./linkis-daemon.sh restart cg-engineplugin ``` diff --git a/docs/zh_CN/ch1/exchangis_user_manual_cn.md b/docs/zh_CN/ch1/exchangis_user_manual_cn.md index 1e91bf9bf..1e7c3c2cc 100644 --- a/docs/zh_CN/ch1/exchangis_user_manual_cn.md +++ b/docs/zh_CN/ch1/exchangis_user_manual_cn.md @@ -34,7 +34,7 @@ ![datasource_list](../../../images/zh_CN/ch1/datasource_list.png)
-图4-1 数据源管理列表 +图3-1 数据源管理列表
@@ -45,7 +45,7 @@ ![datasource_type](../../../images/zh_CN/ch1/datasource_type.png)
-图4-2 数据源类型 +图3-2 数据源类型
@@ -54,7 +54,7 @@ ![MySQL_datasource_config](../../../images/zh_CN/ch1/MySQL_datasource_config.png)
-图4-3 MySQL数据源配置 +图3-3 MySQL数据源配置
@@ -63,7 +63,7 @@ ![Hive_datasource_config](../../../images/zh_CN/ch1/Hive_datasource_config.png)
-图4-4 Hive数据源配置 +图3-4 Hive数据源配置
@@ -74,14 +74,14 @@ ![datasource_func](../../../images/zh_CN/ch1/datasource_func.png)
-图4-5 数据源发布功能 +图3-5 数据源发布功能
  数据源管理的**过期**功能，用于提示此数据源已经逐渐要被替换，请及时更换使用该数据源的任务配置，避免直接删除数据源造成所配置的执行任务失效。 ![datasource_timelimit](../../../images/zh_CN/ch1/datasource_timelimit.png)
-图4-6 数据源过期功能 +图3-6 数据源过期功能
## 四、项目管理 @@ -93,29 +93,46 @@ ![item_list](../../../images/zh_CN/ch1/item_list.png)
-图3-1 项目列表 +图4-1 项目列表
- ### 2、任务列表 +进入项目，可以看到该项目下的任务列表。 + +#### 1）任务管理 +   任务列表中可以对创建的Job数据同步任务进行管理，与项目类似，包括**创建**，**修改**、**删除**和**搜索**。 ![job_task_list](../../../images/zh_CN/ch1/job_task_list.png)
-图3-2 任务列表 +图4-2 任务列表
-  除此之外，**任务支持复制**，能够增加需要的任务，复制的任务包含其原任务配置的所有信息。点击**创建任务**，能够选择任务类型和执行引擎，**目前仅支持离线任务和SQOOP执行引擎**，未来将会支持流式任务和DataX引擎等。 +  点击**创建任务**，能够选择任务类型和执行引擎，**目前仅支持离线任务和SQOOP执行引擎**，未来将会支持流式任务和DataX引擎等。 ![task_type_and_engine](../../../images/zh_CN/ch1/task_type_and_engine.png)
-图3-3 任务类型和引擎配置 -
+图4-3 任务类型和引擎配置 + + +#### 2）子任务管理 + +  在任务中点击**添加子任务**，可以添加多个子任务，支持对子任务的**修改、复制和删除** + +  同时也支持**任务支持复制**，复制的子任务包含其原子任务配置的所有信息。 + +![1656571126825](../../../images/zh_CN/ch1/sub_task_manage.png) -### 3、数据同步任务配置和执行 +
+图4-4 子任务管理 +
+ +#### 3）数据同步任务配置和执行 -  数据同步任务配置和执行是Exchangis1.0的核心功能，基本配置数据同步流程为：**添加子任务->选择源数据源库表和目的数据源库表（在数据源管理模块添加好数据源以供选择）->字段映射（可默认）->过程控制（可默认）->配置（可默认）->执行**。 +  进行该步骤前，需要预先在**数据源管理模块**添加好数据源以供选择，并进行**发布**，当前Exchangis版本仅支持**MySQL数据源和Hive数据源**。 + +  数据同步任务配置和执行是Exchangis1.0.0的核心功能，基本配置数据同步流程为：**添加子任务 -> 选择Source数据源和Sink数据源 -> 字段映射配置 -> 过程控制 -> 任务配置 -> 保存 -> 执行**。 任务执行主要功能包括： 1. 子任务卡片的添加，复制和删除； @@ -128,30 +145,49 @@ 8. 任务执行历史状态查看； 9. 执行任务kill操作。 -### 4、数据源选择和配置 +##### 数据源选择和配置 + +  对于新创建的数据同步子任务，要进行数据源库表的选择，选择的数据源为在**数据源模块**中已发布的数据源。数据源选择支持搜索，搜索方式为先搜索库，再搜索表。 -  点击**添加子任务**开始创建一个数据同步任务，对于新创建的数据同步子任务，首先要进行数据源库表的选择，数据源要在**数据源管理模块**中提前配置好，才会在任务配置中出现。数据源选择支持搜索，搜索方式为先搜索库，再搜索表。 +  MySQL为目的地数据源时，支持**插入**和**更新**两种写入方式；为源数据源时，支持**WHERE条件语句查询**。 + +  Hive为目的地数据源时，支持分区信息配置，写入方式为**追加数据**和**覆盖**两种；为源数据源时，支持**分区信息配置**。 -![add_subtask](../../../images/zh_CN/ch1/add_subtask.png) +![add_subtask](../../../images/zh_CN/ch1/data_source_select.png)
-图3-4 添加子任务 +图4-5 选择数据源
-  MySQL为目的地数据源时，支持**插入**和**更新**两种写入方式；为源数据源时，支持**WHERE条件语句查询**。 + ![1656574588669](../../../images/zh_CN/ch1/data_source_insert_way.png) -  Hive为目的地数据源时，支持分区信息配置，写入方式为**追加数据**和**覆盖**两种；为源数据源时，支持**分区信息配置**。 +
+图4-6 写入方式配置 +
+ +##### 数据源字段映射 -### 5、数据源字段映射 +  当配置完成数据源库表信息时，Exchangis1.0.0会自动进行Source数据源和Sink数据源的字段映射，并且可以自行选择想要映射的字段，也可以用来检查我们的字段是否匹配；当Hive为Sink数据源时，其映射字段不可修改。 -  当配置完成数据源库表信息时，Exchangis1.0会自动在**字段映射**一行生成原数据源和目的数据源的字段自动映射，并且可以自行选择想要映射的字段；当HIVE为目的地数据源时，其映射字段不可修改，这点要注意。 +![1656574253553](../../../images/zh_CN/ch1/data_source_field_mapping.png) -### 6、过程控制 +
+图4-7 字段映射 +
-  任务执行提供对任务的最大并行数配置，以及作业最大内存配置，可根据实际需要进行更改。 +##### 过程控制 -### 7、作业执行 +  任务执行提供**作业最大并行数**配置（默认为1个），以及**作业最大内存**配置（默认为1024Mb），可根据实际需要进行更改。 + +![1656574125954](../../../images/zh_CN/ch1/task_proccess_control.png) + +
+图4-8 过程控制 +
+ +#### 4）作业执行 + +  Exchangis1.0支持多个子任务同时执行，任务配置完成后，点击执行，即开始数据同步任务，界面下方会弹出工作台，工作台主要包含三个部分功能：**运行情况、实时日志和执行历史**。 -  Exchangis1.0支持多个子任务同时执行，任务配置完成后，点击执行，即开始数据同步任务，界面下方会弹出工作台，工作台主要包含三个部分功能：**运行情况、实时日志和执行历史**；   **运行情况**能够查看当前数据同步任务整体进度，包含task成功和失败的数量等，以及点击task的名字，能够展示每个task的各项运行指标信息。   **实时日志**主要展示的内容包含两大类，一是整个导数job的日志，能够输出每个task的状态日志，例如task是否被调度，是否运行中等；二是每个task的日志，输出的是各自相应的导数日志。在实时日志中能够根据关键字和忽略字进行日志筛选，并别提供获取最后n行日志功能；还可以对Error、Warning和Info不同类型的日志进行筛选展示，只需点击相应的按钮即可。 @@ -260,3 +296,7 @@ Exchangis Appconn主要支持以下功能： ![production_center](../../../images/zh_CN/ch1/production_center.png) +
+图6-8 生产中心 +
+ diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java index 79b75354e..0cb3ee3cd 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java @@ -64,8 +64,8 @@ public Message importProject(HttpServletRequest req, Map params) Long projectId = Long.parseLong("111111"); String projectVersion = (String) params.get("projectVersion"); String flowVersion = (String) params.get("flowVersion"); - String versionSuffix = projectVersion; - LOG.info("resourceId: {}, projectId: {}, versionSuffix: {}, version: {}, userName: {}", resourceId, projectId, versionSuffix, version, userName); + String versionSuffix = projectVersion + "_" + flowVersion; + LOG.info("resourceId: {}, projectId: {}, versionSuffix: {}, version: {}, userName: {}, flowVersion: {}", resourceId, projectId, versionSuffix, version, userName, flowVersion); BmlClient bmlClient = BmlClientFactory.createBmlClient(userName); BmlDownloadResponse bmlDownloadResponse = bmlClient.downloadShareResource(userName, resourceId, version); LOG.info("bmlDownloadResponse: {}", bmlDownloadResponse); diff --git a/exchangis-plugins/engine/sqoop/pom.xml b/exchangis-plugins/engine/sqoop/pom.xml index f475aa57a..345ae5443 100644 --- a/exchangis-plugins/engine/sqoop/pom.xml +++ b/exchangis-plugins/engine/sqoop/pom.xml @@ -23,7 +23,7 @@ linkis org.apache.linkis 1.1.1 - ../../pom.xml + 4.0.0 diff --git a/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml b/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml index 8383286c3..9174d8a49 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml +++ b/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml @@ -69,6 +69,15 @@ db unix + + ${basedir}/src/main/resources + + sqoop.icon + + 0777 + icons + unix + diff --git a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql index b33827a80..c2c70b0b7 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql +++ b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql @@ -18,6 +18,7 @@ INSERT INTO `dss_workspace_menu_appconn` (`appconn_id`, `menu_id`, `title_en`, ` VALUES(@dss_appconn_exchangisId,@exchangis_menuId,'Exchangis','Exchangis','Exchangis','' ,'exchangis, statement','数据交换,数据源','1','enter Exchangis','进入Exchangis','user manual','用户手册','/manual_url','shujukeshihua-logo',NULL,NULL,NULL,NULL,NULL,'shujukeshihua-icon'); +select @dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'; delete from `dss_workflow_node` where `node_type` like '%exchangis%'; insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('sqoop','exchangis','linkis.appconn.exchangis.sqoop',1,'1','1','0','1','icons/sqoop.icon'); diff --git a/exchangis-plugins/exchangis-appconn/src/main/resources/sqoop.icon b/exchangis-plugins/exchangis-appconn/src/main/resources/sqoop.icon new file mode 100644 index 000000000..bf047fb7b --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/resources/sqoop.icon @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/exchangis-plugins/pom.xml b/exchangis-plugins/pom.xml index 578ffc5bc..1f7ed71d1 100644 --- a/exchangis-plugins/pom.xml +++ b/exchangis-plugins/pom.xml @@ -14,6 +14,7 @@ 1.0.0 exchangis-appconn + engine/sqoop diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java index 781e63a15..46253968d 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java @@ -53,7 +53,6 @@ public Message createProject(@Validated @RequestBody ExchangisProjectAppVo proje } long projectIdd = projectService.createProject(projectVo, username); String projectId = String.valueOf(projectIdd); - System.out.println(projectId); return ExchangisProjectRestfulUtils.dealOk("创建工程成功", new Pair<>("projectName", projectVo.getName()), new Pair<>("projectId", projectId)); diff --git a/images/zh_CN/ch1/add_subtask.png b/images/zh_CN/ch1/add_subtask.png deleted file mode 100644 index e3b01ae29..000000000 Binary files a/images/zh_CN/ch1/add_subtask.png and /dev/null differ diff --git a/images/zh_CN/ch1/data_source_field_mapping.png b/images/zh_CN/ch1/data_source_field_mapping.png new file mode 100644 index 000000000..53d139080 Binary files /dev/null and b/images/zh_CN/ch1/data_source_field_mapping.png differ diff --git a/images/zh_CN/ch1/data_source_insert_way.png b/images/zh_CN/ch1/data_source_insert_way.png new file mode 100644 index 000000000..1cfad3578 Binary files /dev/null and b/images/zh_CN/ch1/data_source_insert_way.png differ diff --git a/images/zh_CN/ch1/data_source_select.png b/images/zh_CN/ch1/data_source_select.png new file mode 100644 index 000000000..11c813d67 Binary files /dev/null and b/images/zh_CN/ch1/data_source_select.png differ diff --git a/images/zh_CN/ch1/sub_task_manage.png b/images/zh_CN/ch1/sub_task_manage.png new file mode 100644 index 000000000..904954aab Binary files /dev/null and b/images/zh_CN/ch1/sub_task_manage.png differ diff --git a/images/zh_CN/ch1/task_proccess_control.png b/images/zh_CN/ch1/task_proccess_control.png new file mode 100644 index 000000000..69d1089ec Binary files /dev/null and b/images/zh_CN/ch1/task_proccess_control.png differ diff --git a/pom.xml b/pom.xml index 7447927c7..0d9b94e1a 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ - 1.0.0-RC1 + 1.0.0 1.1.0 1.1.1 0.1.0-SNAPSHOT