From 2fc0db20ac5357f804081776d1b590fbd034e3ee Mon Sep 17 00:00:00 2001 From: jefftlin Date: Mon, 24 Oct 2022 19:18:19 +0800 Subject: [PATCH 01/20] Fix query error with character "_" --- .../datasource/service/ExchangisDataSourceService.java | 2 +- .../server/restful/execute/ExchangisJobExecuteRestfulApi.java | 1 + .../exchangis/project/server/mapper/impl/ProjectMapper.xml | 2 +- .../project/server/restful/ExchangisProjectRestfulApi.java | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java index 9527aff11..7b313c381 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java @@ -630,7 +630,7 @@ public Message queryDataSources(HttpServletRequest request, DataSourceQueryVO vo Integer page = Objects.isNull(vo.getPage()) ? 1 : vo.getPage(); Integer pageSize = Objects.isNull(vo.getPageSize()) ? 100 : vo.getPageSize(); - String dataSourceName = Objects.isNull(vo.getName()) ? "" : vo.getName(); + String dataSourceName = Objects.isNull(vo.getName()) ? "" : vo.getName().replace("_", "\\_"); LinkisDataSourceRemoteClient linkisDataSourceRemoteClient = ExchangisLinkisRemoteClient.getLinkisDataSourceRemoteClient(); QueryDataSourceResult result; int totalPage = 0; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java index 4e1b84886..5ed4cfb4a 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java @@ -218,6 +218,7 @@ public Message listJobs(@RequestParam(value = "jobExecutionId", required = false @RequestParam(value = "size", required = false) int size, HttpServletRequest request) { Message message = Message.ok("Submitted succeed(提交成功)！"); + jobName = jobName.replace("_", "\\_"); try { List jobList = executeService.getExecutedJobList(jobExecutionId, jobName, status, launchStartTime, launchEndTime, current, size, request); diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml index df73890be..5fc2ca755 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml @@ -33,7 +33,7 @@ - AND dp.name like concat('%', #{name}, '%') escape '/' + AND dp.name like concat('%', #{name}, '%') AND dp.domain = #{domain} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java index 8d0c5de1c..5316a97a0 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java @@ -65,10 +65,10 @@ public Message queryProjects(HttpServletRequest request, @RequestParam(value = "size", required = false) Integer size, @RequestParam(value = "name", required = false) String name) { String username = UserUtils.getLoginUser(request); - String newName = name.replaceAll("_", "/_"); + name = name.replace("_", "\\_"); Optional.ofNullable(current).ifPresent(queryVo::setCurrent); Optional.ofNullable(size).ifPresent(queryVo::setSize); - Optional.ofNullable(newName).ifPresent(queryVo::setName); + Optional.ofNullable(name).ifPresent(queryVo::setName); queryVo.setCreateUser(username); try { PageResult pageResult = projectService.queryProjects(queryVo); From d43060055776e78cd4dd78648976ac884dcaf789 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Mon, 31 Oct 2022 19:30:20 +0800 Subject: [PATCH 02/20] Fix the bug that contains spaces in hdfs-writer of datax --- .../builder/transform/mappings/HiveDataxParamsMapping.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java index 8d714fdb5..031425be7 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java @@ -199,7 +199,7 @@ private enum Type { } path = pathBuilder.toString(); } - return path; + return path.replaceAll(" ", "%20"); }); /** From e33924338e2e7d597a7231a81b774fda12d65cc1 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Mon, 31 Oct 2022 19:40:20 +0800 Subject: [PATCH 03/20] Fix query projects --- .../server/restful/ExchangisProjectRestfulApi.java | 9 +++++---- .../exchangis/project/server/vo/ProjectQueryVo.java | 5 +++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java index 5316a97a0..c6fff1573 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java @@ -55,17 +55,18 @@ public class ExchangisProjectRestfulApi { * @param queryVo query vo * @param current current page * @param size size - * @param name name * @return message */ @RequestMapping( value = "projects", method = {RequestMethod.POST, RequestMethod.GET}) public Message queryProjects(HttpServletRequest request, @RequestBody ProjectQueryVo queryVo, @RequestParam(value = "current", required = false) Integer current, - @RequestParam(value = "size", required = false) Integer size, - @RequestParam(value = "name", required = false) String name) { + @RequestParam(value = "size", required = false) Integer size) { String username = UserUtils.getLoginUser(request); - name = name.replace("_", "\\_"); + String name = queryVo.getName(); + if (StringUtils.isNotBlank(name)) { + name = name.replaceAll("_", "/_"); + } Optional.ofNullable(current).ifPresent(queryVo::setCurrent); Optional.ofNullable(size).ifPresent(queryVo::setSize); Optional.ofNullable(name).ifPresent(queryVo::setName); diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java index 7f56cc0f5..51d5a4a99 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java @@ -31,4 +31,9 @@ public ProjectQueryVo(String name, Integer current, Integer size){ this.size = Optional.ofNullable(size).orElse(10); } + public ProjectQueryVo(Map labels, String name){ + this.labels = labels; + this.name = name; + } + } From ac381d8f57256555d3a142ca9bd4575ac747e475 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Fri, 19 May 2023 16:18:32 +0800 Subject: [PATCH 04/20] Update init.sql --- .../src/main/resources/init.sql | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql index a1eae0547..97a567232 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql +++ b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql @@ -18,20 +18,34 @@ INSERT INTO `dss_workspace_menu_appconn` (`appconn_id`, `menu_id`, `title_en`, ` VALUES(@dss_appconn_exchangis_id,@exchangis_menuId,'Exchangis','Exchangis','Exchangis','' ,'exchangis, statement','数据交换,数据源','1','enter Exchangis','进入Exchangis','user manual','用户手册','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/#/projectManage','shujukeshihua-logo',NULL,NULL,NULL,NULL,NULL,'shujukeshihua-icon'); -select @dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'; +-- Sqoop节点安装 +select @old_dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'; +select @old_dss_exchangis_dataxId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'; delete from `dss_workflow_node` where `node_type` like '%exchangis%'; +delete from `dss_workflow_node_to_group` where `node_id`=@old_dss_exchangis_sqoopId or `node_id`=@old_dss_exchangis_dataxId; +delete from `dss_workflow_node_to_ui` where `workflow_node_id`=@old_dss_exchangis_sqoopId or `workflow_node_id`=@old_dss_exchangis_dataxId; + +-- 节点表dss_workflow_node insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('sqoop','exchangis','linkis.appconn.exchangis.sqoop',1,'1','1','0','1','icons/sqoop.icon'); - --- 查找节点所属组的id select @dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'; -delete from `dss_workflow_node_to_group` where `node_id`=@dss_exchangis_sqoopId; -INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@dss_exchangis_sqoopId, 1); +insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) +values('exchangis','exchangis','linkis.appconn.exchangis.datax',1,'1','1','0','1','icons/datax.icon'); +select @dss_exchangis_dataxId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'; + +-- 节点组表dss_workflow_node_to_group +select @dss_workflow_node_group_id:=id from `dss_workflow_node_group` where `name` = '数据交换'; +INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@dss_exchangis_sqoopId, @dss_workflow_node_group_id); +INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@dss_exchangis_dataxId, @dss_workflow_node_group_id); --- 查找sqoop节点所属组的id -delete from `dss_workflow_node_to_ui` where `workflow_node_id`=@dss_exchangis_sqoopId; +-- 节点UI表dss_workflow_node_to_ui INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId, 1); INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId, 2); INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId, 3); INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId, 4); INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId, 5); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId, 1); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId, 2); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId, 3); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId, 4); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId, 5); From c645117e3a87d51029b74963e42399947e9b5bb5 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Thu, 13 Jul 2023 17:17:47 +0800 Subject: [PATCH 05/20] Add exchangis_datax_deploy_cn.md --- docs/zh_CN/ch1/exchangis_datax_deploy_cn.md | 66 +++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docs/zh_CN/ch1/exchangis_datax_deploy_cn.md diff --git a/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md b/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md new file mode 100644 index 000000000..250384a89 --- /dev/null +++ b/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md @@ -0,0 +1,66 @@ +# DataX 引擎使用文档 +### 环境准备 +DataX引擎是执行Exchangis数据同步任务不可或缺的组件，只有安装部署完成DataX引擎才能够成功执行数据同步任务。同时，确保所部署的机器上有安装DataX。 + +您在安装部署DataX引擎之前，请按照[Exchangis1.1.2安装部署文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.1.2/docs/zh_CN/ch1/exchangis_deploy_cn.md)（TODO 待修改连接地址）安装完成Exchangis1.1.2及相关组件的安装，并确保工程基本功能可用。 + +强烈建议您在执行DataX任务之前，先在该节点使用原生的DataX执行测试任务，以检测该节点环境是否正常。 + +| 环境变量名 | 环境变量内容 | 备注 | +| :----: | :----: |-------| +| JAVA_HOME | JDK安装路径 | 必须 | +| DataX_HOME | DataX安装路径 | 非必须 | +| DataX_CONF_DIR | DataX配置路径 | 非必须 | + +### 安装包准备 +#### 1）下载二进制包 + +Exchangis1.1.2和Linkis 1.4.0支持的主流DataX版本1.4.6与1.4.7，更高版本可能需要修改部分代码重新编译。 + +[点击跳转 Release 界面](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.1.2)（TODO 待修改连接地址） + +#### 2） 编译打包 +如果您想自己开发和编译datax引擎，具体编译步骤如下： + +1、克隆Exchangis的代码 + +2、在exchangis-plugins模块下，找到datax引擎，单独编译datax，操作如下 + +``` +cd {EXCHANGIS_CODE_HOME}/exchangis-plugins/engine/datax +mvn clean install +``` +然后会在该路径下找到datax引擎安装包 +``` +{EXCHANGIS_CODE_HOME}/exchangis-plugins/datax/target/out +``` + + +### 开始部署 +#### 1）DataX引擎安装 +1、拿到打包出来的datax物料包，目录结构为： + +```shell +datax +-- dist +-- plugin +``` + +2、放置到linkis安装路径的如下目录 + +```shell +cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins +``` +（注意，看当前datax引擎对哪些用户有权限，一般都为hadoop用户组和hadoop用户） + + +#### 2）重启linkis-engineplugin服务使datax引擎生效 +新加入linkis的引擎都要重启linkis的engineplugin服务才会生效，重启脚本为linkis安装目录下的./sbin/linkis-daemon.sh，具体步骤如下 +```shell +cd {LINKIS_INSTALL_HOME}/links/sbin/ +./linkis-daemon.sh restart cg-engineplugin +``` +待服务启动成功，至此，datax安装部署就完成了。 + +engineplugin更详细的介绍可以参看下面的文章。 +https://linkis.apache.org/zh-CN/docs/latest/deployment/engine-conn-plugin-installation/ \ No newline at end of file From fcf2d0e0cc88ce8d99a5ec96bcd5568f826c15f0 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Thu, 13 Jul 2023 17:18:11 +0800 Subject: [PATCH 06/20] Remove unused config file --- assembly-package/config/application-eureka.yml | 0 assembly-package/config/exchangis.properties | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 assembly-package/config/application-eureka.yml delete mode 100644 assembly-package/config/exchangis.properties diff --git a/assembly-package/config/application-eureka.yml b/assembly-package/config/application-eureka.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/assembly-package/config/exchangis.properties b/assembly-package/config/exchangis.properties deleted file mode 100644 index e69de29bb..000000000 From f43db175f5c5471d344ed41366c1e4d905ee365e Mon Sep 17 00:00:00 2001 From: jefftlin Date: Thu, 13 Jul 2023 17:19:05 +0800 Subject: [PATCH 07/20] Upgrade version of netty --- exchangis-engines/engines/datax/datax-hdfsreader/pom.xml | 2 +- exchangis-engines/engines/datax/datax-hdfswriter/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml index 220ed8051..f3ab749a2 100644 --- a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml @@ -61,7 +61,7 @@ io.netty netty-all - 4.1.46.Final + 4.1.86.Final org.apache.hadoop diff --git a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml index e186cb375..a03b65a49 100644 --- a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml @@ -66,7 +66,7 @@ io.netty netty-all - 4.1.46.Final + 4.1.86.Final org.apache.hadoop From cc7727e1e1ac16ddb029f362d7397e2f321a0dc4 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Thu, 13 Jul 2023 17:39:58 +0800 Subject: [PATCH 08/20] Upgrade exchangis to version 1.1.2 --- assembly-package/pom.xml | 2 +- exchangis-dao/pom.xml | 2 +- .../exchangis-datasource-core/pom.xml | 4 ++-- .../exchangis-datasource-linkis/pom.xml | 4 ++-- .../exchangis-datasource-loader/pom.xml | 4 ++-- .../exchangis-datasource-server/pom.xml | 6 +++--- .../exchangis-datasource-service/pom.xml | 8 ++++---- .../exchangis-datasource-streamis/pom.xml | 4 ++-- .../pom.xml | 8 ++++---- .../exchangis-datasource-ext-hive/pom.xml | 8 ++++---- .../exchangis-datasource-ext-mongodb/pom.xml | 8 ++++---- .../exchangis-datasource-ext-mysql/pom.xml | 8 ++++---- .../exchangis-datasource-ext-oracle/pom.xml | 8 ++++---- .../exchangis-datasource-ext-sftp/pom.xml | 8 ++++---- exchangis-datasource/pom.xml | 3 +-- .../engineconn-plugins/datax/pom.xml | 2 +- .../engineconn-plugins/sqoop/pom.xml | 2 +- .../engines/datax/datax-assembly/pom.xml | 2 +- .../engines/datax/datax-core/pom.xml | 2 +- .../datax/datax-elasticsearchwriter/pom.xml | 2 +- .../engines/datax/datax-ftpreader/pom.xml | 2 +- .../engines/datax/datax-ftpwriter/pom.xml | 2 +- .../engines/datax/datax-hdfsreader/pom.xml | 2 +- .../engines/datax/datax-hdfswriter/pom.xml | 2 +- .../engines/datax/datax-mysqlreader/pom.xml | 2 +- .../engines/datax/datax-mysqlwriter/pom.xml | 2 +- .../engines/datax/datax-oraclereader/pom.xml | 2 +- .../engines/datax/datax-oraclewriter/pom.xml | 2 +- .../datax/datax-textfilereader/pom.xml | 2 +- .../datax/datax-textfilewriter/pom.xml | 2 +- exchangis-engines/engines/datax/pom.xml | 2 +- .../exchangis-engine-common/pom.xml | 2 +- .../exchangis-engine-core/pom.xml | 2 +- .../exchangis-engine-server/pom.xml | 2 +- exchangis-engines/pom.xml | 4 ++-- exchangis-job/exchangis-job-builder/pom.xml | 6 +++--- exchangis-job/exchangis-job-common/pom.xml | 6 +++--- exchangis-job/exchangis-job-launcher/pom.xml | 6 +++--- exchangis-job/exchangis-job-metrics/pom.xml | 2 +- exchangis-job/exchangis-job-server/pom.xml | 20 ++++++++++--------- exchangis-job/pom.xml | 4 ++-- exchangis-plugins/exchangis-appconn/pom.xml | 2 +- exchangis-plugins/pom.xml | 4 ++-- .../exchangis-project-server/pom.xml | 9 +++------ exchangis-project/pom.xml | 3 +-- exchangis-server/pom.xml | 17 +++++----------- pom.xml | 4 ++-- 47 files changed, 100 insertions(+), 110 deletions(-) diff --git a/assembly-package/pom.xml b/assembly-package/pom.xml index d8b300d5b..45315a5fe 100644 --- a/assembly-package/pom.xml +++ b/assembly-package/pom.xml @@ -21,7 +21,7 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 assembly-package diff --git a/exchangis-dao/pom.xml b/exchangis-dao/pom.xml index 43714347a..c568837c6 100644 --- a/exchangis-dao/pom.xml +++ b/exchangis-dao/pom.xml @@ -5,7 +5,7 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 diff --git a/exchangis-datasource/exchangis-datasource-core/pom.xml b/exchangis-datasource/exchangis-datasource-core/pom.xml index ecca5a44f..990c92d6f 100644 --- a/exchangis-datasource/exchangis-datasource-core/pom.xml +++ b/exchangis-datasource/exchangis-datasource-core/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -20,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/exchangis-datasource-linkis/pom.xml b/exchangis-datasource/exchangis-datasource-linkis/pom.xml index 4acb1d9b8..2ab7eda3d 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/pom.xml +++ b/exchangis-datasource/exchangis-datasource-linkis/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -20,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/exchangis-datasource-loader/pom.xml b/exchangis-datasource/exchangis-datasource-loader/pom.xml index 91978eec3..30ebb07ad 100644 --- a/exchangis-datasource/exchangis-datasource-loader/pom.xml +++ b/exchangis-datasource/exchangis-datasource-loader/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -20,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/exchangis-datasource-server/pom.xml b/exchangis-datasource/exchangis-datasource-server/pom.xml index f759adf04..c41abaa25 100644 --- a/exchangis-datasource/exchangis-datasource-server/pom.xml +++ b/exchangis-datasource/exchangis-datasource-server/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -20,13 +20,13 @@ com.webank.wedatasphere.exchangis exchangis-datasource-service - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-loader - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/exchangis-datasource-service/pom.xml b/exchangis-datasource/exchangis-datasource-service/pom.xml index 157270d7b..e41e65f03 100644 --- a/exchangis-datasource/exchangis-datasource-service/pom.xml +++ b/exchangis-datasource/exchangis-datasource-service/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -21,19 +21,19 @@ com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-engine-core - 1.1.1 + 1.1.2 compile diff --git a/exchangis-datasource/exchangis-datasource-streamis/pom.xml b/exchangis-datasource/exchangis-datasource-streamis/pom.xml index a3695e549..906e781a2 100644 --- a/exchangis-datasource/exchangis-datasource-streamis/pom.xml +++ b/exchangis-datasource/exchangis-datasource-streamis/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -20,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml index 6ea20452a..a92fa201c 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml index ebc851eb6..c1e258f85 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -22,17 +22,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml index 60f6b2afd..a26312d6e 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml index 6201fc9e2..71bb30072 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml index 7d6288c5b..e3cce71da 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml index d02a5c64d..47af538ab 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/pom.xml b/exchangis-datasource/pom.xml index ce24d510b..ad1bd9fae 100644 --- a/exchangis-datasource/pom.xml +++ b/exchangis-datasource/pom.xml @@ -5,13 +5,12 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 exchangis-datasource pom - 1.1.1 exchangis-datasource-core diff --git a/exchangis-engines/engineconn-plugins/datax/pom.xml b/exchangis-engines/engineconn-plugins/datax/pom.xml index 1da0bb390..13c16c435 100644 --- a/exchangis-engines/engineconn-plugins/datax/pom.xml +++ b/exchangis-engines/engineconn-plugins/datax/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 diff --git a/exchangis-engines/engineconn-plugins/sqoop/pom.xml b/exchangis-engines/engineconn-plugins/sqoop/pom.xml index 65aaab13b..7c8f8cde5 100644 --- a/exchangis-engines/engineconn-plugins/sqoop/pom.xml +++ b/exchangis-engines/engineconn-plugins/sqoop/pom.xml @@ -22,7 +22,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-assembly/pom.xml b/exchangis-engines/engines/datax/datax-assembly/pom.xml index 560ecbc90..74b755512 100644 --- a/exchangis-engines/engines/datax/datax-assembly/pom.xml +++ b/exchangis-engines/engines/datax/datax-assembly/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-core/pom.xml b/exchangis-engines/engines/datax/datax-core/pom.xml index 78134c869..491136a65 100644 --- a/exchangis-engines/engines/datax/datax-core/pom.xml +++ b/exchangis-engines/engines/datax/datax-core/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml b/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml index e844cdf77..357a9b855 100644 --- a/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-ftpreader/pom.xml b/exchangis-engines/engines/datax/datax-ftpreader/pom.xml index 6e1a4d70a..8520f0145 100644 --- a/exchangis-engines/engines/datax/datax-ftpreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-ftpreader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml b/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml index 37a55d81f..ec0635c29 100644 --- a/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml index f3ab749a2..737641292 100644 --- a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml index a03b65a49..a8f25054b 100644 --- a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml b/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml index a72c46afd..12389b823 100644 --- a/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml b/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml index dbf27f6de..7339dbc27 100644 --- a/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-oraclereader/pom.xml b/exchangis-engines/engines/datax/datax-oraclereader/pom.xml index 9754669a9..fd6f86f07 100644 --- a/exchangis-engines/engines/datax/datax-oraclereader/pom.xml +++ b/exchangis-engines/engines/datax/datax-oraclereader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml b/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml index 82db85b0c..054951019 100644 --- a/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-textfilereader/pom.xml b/exchangis-engines/engines/datax/datax-textfilereader/pom.xml index 452119ba6..6e7c18313 100644 --- a/exchangis-engines/engines/datax/datax-textfilereader/pom.xml +++ b/exchangis-engines/engines/datax/datax-textfilereader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml b/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml index 6e1a746bd..1d0c9f1f3 100644 --- a/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/pom.xml b/exchangis-engines/engines/datax/pom.xml index ff2abca29..3cef763af 100644 --- a/exchangis-engines/engines/datax/pom.xml +++ b/exchangis-engines/engines/datax/pom.xml @@ -5,7 +5,7 @@ com.webank.wedatasphere.exchangis exchangis - 1.1.1 + 1.1.2 ../../../pom.xml diff --git a/exchangis-engines/exchangis-engine-common/pom.xml b/exchangis-engines/exchangis-engine-common/pom.xml index 1501f3206..d6cf14efa 100644 --- a/exchangis-engines/exchangis-engine-common/pom.xml +++ b/exchangis-engines/exchangis-engine-common/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 diff --git a/exchangis-engines/exchangis-engine-core/pom.xml b/exchangis-engines/exchangis-engine-core/pom.xml index a551b75e6..5d040b341 100644 --- a/exchangis-engines/exchangis-engine-core/pom.xml +++ b/exchangis-engines/exchangis-engine-core/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 diff --git a/exchangis-engines/exchangis-engine-server/pom.xml b/exchangis-engines/exchangis-engine-server/pom.xml index fb7370bb6..5873fe538 100644 --- a/exchangis-engines/exchangis-engine-server/pom.xml +++ b/exchangis-engines/exchangis-engine-server/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 diff --git a/exchangis-engines/pom.xml b/exchangis-engines/pom.xml index 783dbaac0..7dfac3c51 100644 --- a/exchangis-engines/pom.xml +++ b/exchangis-engines/pom.xml @@ -5,13 +5,13 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 exchangis-engines pom - 1.1.1 + 1.1.2 exchangis-engine-common diff --git a/exchangis-job/exchangis-job-builder/pom.xml b/exchangis-job/exchangis-job-builder/pom.xml index 18e6e36e1..acfbb460a 100644 --- a/exchangis-job/exchangis-job-builder/pom.xml +++ b/exchangis-job/exchangis-job-builder/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -15,7 +15,7 @@ com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.1 + 1.1.2 com.google.code.gson @@ -25,7 +25,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-service - 1.1.1 + 1.1.2 compile diff --git a/exchangis-job/exchangis-job-common/pom.xml b/exchangis-job/exchangis-job-common/pom.xml index a15fc587c..1bae2fa82 100644 --- a/exchangis-job/exchangis-job-common/pom.xml +++ b/exchangis-job/exchangis-job-common/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -15,7 +15,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 org.apache.linkis @@ -25,7 +25,7 @@ com.webank.wedatasphere.exchangis exchangis-engine-common - 1.1.1 + 1.1.2 org.apache.linkis diff --git a/exchangis-job/exchangis-job-launcher/pom.xml b/exchangis-job/exchangis-job-launcher/pom.xml index 88aabb8d6..2375cdea3 100644 --- a/exchangis-job/exchangis-job-launcher/pom.xml +++ b/exchangis-job/exchangis-job-launcher/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -15,12 +15,12 @@ com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-job-builder - 1.1.1 + 1.1.2 org.apache.linkis diff --git a/exchangis-job/exchangis-job-metrics/pom.xml b/exchangis-job/exchangis-job-metrics/pom.xml index 52b3f3f73..9dcdd12fb 100644 --- a/exchangis-job/exchangis-job-metrics/pom.xml +++ b/exchangis-job/exchangis-job-metrics/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 diff --git a/exchangis-job/exchangis-job-server/pom.xml b/exchangis-job/exchangis-job-server/pom.xml index 3a370b347..5f69ba57c 100644 --- a/exchangis-job/exchangis-job-server/pom.xml +++ b/exchangis-job/exchangis-job-server/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -17,24 +17,26 @@ - com.webank.wedatasphere.exchangis - exchangis-job-common - 1.1.1 + exchangis-project-server + 1.1.2 - com.webank.wedatasphere.exchangis - exchangis-datasource-service - 1.1.1 + exchangis-job-common + 1.1.2 com.webank.wedatasphere.exchangis exchangis-job-launcher - 1.1.1 + 1.1.2 + + + com.webank.wedatasphere.exchangis + exchangis-datasource-service + 1.1.2 - com.webank.wedatasphere.exchangis exchangis-engine-core diff --git a/exchangis-job/pom.xml b/exchangis-job/pom.xml index 4a33a7154..e28c390ac 100644 --- a/exchangis-job/pom.xml +++ b/exchangis-job/pom.xml @@ -5,13 +5,13 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 exchangis-job pom - 1.1.1 + 1.1.2 exchangis-job-common diff --git a/exchangis-plugins/exchangis-appconn/pom.xml b/exchangis-plugins/exchangis-appconn/pom.xml index 98d291035..0c1212479 100644 --- a/exchangis-plugins/exchangis-appconn/pom.xml +++ b/exchangis-plugins/exchangis-appconn/pom.xml @@ -5,7 +5,7 @@ exchangis-plugins com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-plugins/pom.xml b/exchangis-plugins/pom.xml index 486e5d4a0..8b290ca55 100644 --- a/exchangis-plugins/pom.xml +++ b/exchangis-plugins/pom.xml @@ -5,13 +5,13 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 exchangis-plugins pom - 1.1.1 + 1.1.2 exchangis-appconn diff --git a/exchangis-project/exchangis-project-server/pom.xml b/exchangis-project/exchangis-project-server/pom.xml index 83cd118dc..c0e02e2a6 100644 --- a/exchangis-project/exchangis-project-server/pom.xml +++ b/exchangis-project/exchangis-project-server/pom.xml @@ -5,7 +5,7 @@ exchangis-project com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -22,7 +22,6 @@ linkis-mybatis ${linkis.version} - org.apache.commons commons-math3 @@ -31,14 +30,12 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 - + 1.1.2 - com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.1 + 1.1.2 diff --git a/exchangis-project/pom.xml b/exchangis-project/pom.xml index 3e7eafdfe..4b7f2e1ac 100644 --- a/exchangis-project/pom.xml +++ b/exchangis-project/pom.xml @@ -5,13 +5,12 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 exchangis-project pom - 1.1.1 exchangis-project-server diff --git a/exchangis-server/pom.xml b/exchangis-server/pom.xml index 9873f1ce9..e805d9559 100644 --- a/exchangis-server/pom.xml +++ b/exchangis-server/pom.xml @@ -5,7 +5,7 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -20,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-server - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis @@ -36,31 +36,24 @@ com.webank.wedatasphere.exchangis exchangis-job-server - 1.1.1 + 1.1.2 - com.webank.wedatasphere.exchangis exchangis-project-server - 1.1.1 + 1.1.2 - - - - - org.apache.linkis linkis-module - diff --git a/pom.xml b/pom.xml index 0668720a4..842bc08ca 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ com.webank.wedatasphere.exchangis exchangis - 1.1.1 + 1.1.2 pom exchangis @@ -37,7 +37,7 @@ - 1.1.1 + 1.1.2 1.1.0 1.1.6-webank 1.1.1 From 330af95a40d1fea3c462021e11a5134f21ff978d Mon Sep 17 00:00:00 2001 From: jefftlin Date: Thu, 13 Jul 2023 22:26:22 +0800 Subject: [PATCH 09/20] Adjust the relationship between the exchangis-project module and exchangis-job module --- .../exchangis/common/AuditLogUtils.java | 2 +- ...rceMapper.xml => ProjectExportService.xml} | 0 exchangis-job/exchangis-job-server/pom.xml | 13 +- .../mapping/FieldMappingTransformer.java | 2 - .../restful/ExchangisJobRestfulApi.java | 8 +- .../ExchangisJobDssAppConnRestfulApi.java | 87 -------- .../server/service/IProjectExportService.java | 21 -- .../job/server/service/JobInfoService.java | 19 -- .../service/impl/DefaultJobInfoService.java | 176 --------------- .../job/server/utils/JobAuthorityUtils.java | 4 +- exchangis-job/exchangis-job-service/pom.xml | 33 +++ .../exchangis-project-entity/pom.xml | 32 +++ .../entity/domain}/ExchangisProjectUser.java | 2 +- .../project/entity}/domain/OperationType.java | 2 +- .../entity}/domain/ProjectPageQuery.java | 4 +- .../entity}/entity/ExchangisProject.java | 2 +- .../entity}/vo/ExchangisProjectAppVo.java | 4 +- .../entity}/vo/ExchangisProjectInfo.java | 7 +- .../entity}/vo/ExchangisProjectUserVo.java | 5 +- .../project/entity}/vo/ProjectQueryVo.java | 5 +- .../exchangis-project-provider/pom.xml | 27 +++ .../provider}/mapper/ProjectMapper.java | 6 +- .../provider}/mapper/ProjectUserMapper.java | 4 +- .../provider}/mapper/impl/ProjectMapper.xml | 4 +- .../mapper/impl/ProjectUserMapper.xml | 4 +- .../exchangis-project-server/pom.xml | 12 +- .../restful/ExchangisProjectRestfulApi.java | 33 ++- .../ExchangisJobDssAppConnRestfulApi.java | 109 ++++++++++ .../server/service/ProjectCopyService.java | 4 +- .../server/service/ProjectExportService.java | 30 +++ .../server/service/ProjectImportService.java | 5 +- .../server/service/ProjectService.java | 8 +- .../service/impl/ProjectCopyServiceImpl.java | 16 +- .../impl/ProjectExportServiceImpl.java | 202 ++++++++++++++++++ .../service/impl/ProjectImportServerImpl.java | 12 +- .../service/impl/ProjectServiceImpl.java | 14 +- .../server/utils/ProjectAuthorityUtils.java | 4 +- exchangis-project/pom.xml | 2 + 38 files changed, 526 insertions(+), 398 deletions(-) rename exchangis-engines/exchangis-engine-server/src/main/java/com/webank/wedatasphere/exchangis/engine/server/mapper/{EngineResourceMapper.xml => ProjectExportService.xml} (100%) delete mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectExportService.java create mode 100644 exchangis-job/exchangis-job-service/pom.xml create mode 100644 exchangis-project/exchangis-project-entity/pom.xml rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity => exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain}/ExchangisProjectUser.java (95%) rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server => exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity}/domain/OperationType.java (86%) rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server => exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity}/domain/ProjectPageQuery.java (83%) rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server => exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity}/entity/ExchangisProject.java (98%) rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server => exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity}/vo/ExchangisProjectAppVo.java (97%) rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server => exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity}/vo/ExchangisProjectInfo.java (96%) rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server => exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity}/vo/ExchangisProjectUserVo.java (90%) rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server => exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity}/vo/ProjectQueryVo.java (87%) create mode 100644 exchangis-project/exchangis-project-provider/pom.xml rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server => exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider}/mapper/ProjectMapper.java (90%) rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server => exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider}/mapper/ProjectUserMapper.java (85%) rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server => exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider}/mapper/impl/ProjectMapper.xml (97%) rename exchangis-project/{exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server => exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider}/mapper/impl/ProjectUserMapper.xml (92%) create mode 100644 exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisJobDssAppConnRestfulApi.java rename exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectCopyService.java => exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectCopyService.java (83%) create mode 100644 exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectExportService.java rename exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectImportService.java => exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectImportService.java (76%) rename {exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job => exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project}/server/service/impl/ProjectCopyServiceImpl.java (85%) create mode 100644 exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectExportServiceImpl.java rename {exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job => exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project}/server/service/impl/ProjectImportServerImpl.java (95%) diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/AuditLogUtils.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/AuditLogUtils.java index 40a09abed..f42908668 100644 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/AuditLogUtils.java +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/AuditLogUtils.java @@ -31,7 +31,7 @@ public static void printLog(String user, String proxyUser, TargetTypeEnum target //String detailInfo=new Gson().toJson(params); String detailInfo=params.toString(); LOGGER.info("[{}],[{}],[{}],[{}],[{}],[{}],[{}],[{}],[{}]", - new Date(),user, "proxyUser is: " + proxyUser, "Exchangis-1.1.1", targetType.getName(), + new Date(),user, "proxyUser is: " + proxyUser, "Exchangis-1.1.2", targetType.getName(), targetId,targetName,operateType.getName(), detailInfo); } diff --git a/exchangis-engines/exchangis-engine-server/src/main/java/com/webank/wedatasphere/exchangis/engine/server/mapper/EngineResourceMapper.xml b/exchangis-engines/exchangis-engine-server/src/main/java/com/webank/wedatasphere/exchangis/engine/server/mapper/ProjectExportService.xml similarity index 100% rename from exchangis-engines/exchangis-engine-server/src/main/java/com/webank/wedatasphere/exchangis/engine/server/mapper/EngineResourceMapper.xml rename to exchangis-engines/exchangis-engine-server/src/main/java/com/webank/wedatasphere/exchangis/engine/server/mapper/ProjectExportService.xml diff --git a/exchangis-job/exchangis-job-server/pom.xml b/exchangis-job/exchangis-job-server/pom.xml index 5f69ba57c..60c64165e 100644 --- a/exchangis-job/exchangis-job-server/pom.xml +++ b/exchangis-job/exchangis-job-server/pom.xml @@ -19,12 +19,7 @@ com.webank.wedatasphere.exchangis - exchangis-project-server - 1.1.2 - - - com.webank.wedatasphere.exchangis - exchangis-job-common + exchangis-project-provider 1.1.2 @@ -54,12 +49,6 @@ mysql-connector-java 5.1.49 - - com.webank.wedatasphere.exchangis - exchangis-project-server - 1.1.1 - compile - diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java index 1adc16108..6cbbc64a9 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java @@ -17,11 +17,9 @@ import com.webank.wedatasphere.exchangis.job.server.render.transform.field.mapping.match.FieldMatchStrategy; import com.webank.wedatasphere.exchangis.job.server.utils.SpringContextHolder; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java index 94fcf6636..73929af81 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java @@ -17,8 +17,6 @@ import com.webank.wedatasphere.exchangis.job.server.vo.JobFunction; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; -import com.webank.wedatasphere.exchangis.project.server.exception.ExchangisProjectErrorException; -import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.ProxyUserSSOUtils; import org.apache.linkis.server.security.SecurityFilter; @@ -36,7 +34,6 @@ import java.util.List; import java.util.Objects; - /** * The basic controller of Exchangis job */ @@ -52,9 +49,6 @@ public class ExchangisJobRestfulApi { @Resource private JobInfoService jobInfoService; - @Resource - private ProjectService projectService; - @Resource private JobFuncService jobFuncService; @@ -170,7 +164,7 @@ public Message jobFuncList(@PathVariable("tabName") String tabName, public Message createJob( @Validated({InsertGroup.class, Default.class}) @RequestBody ExchangisJobVo exchangisJobVo, BindingResult result, - HttpServletRequest request) throws ExchangisProjectErrorException { + HttpServletRequest request) { if (ExchangisLauncherConfiguration.LIMIT_INTERFACE.getValue()) { return Message.error("You have no permission to create Job (没有创建任务权限)"); } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java index 0ea358388..9b91a5814 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java @@ -9,22 +9,16 @@ import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; import com.webank.wedatasphere.exchangis.job.domain.OperationType; import com.webank.wedatasphere.exchangis.job.launcher.ExchangisLauncherConfiguration; -import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.job.server.service.IProjectCopyService; import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; import com.webank.wedatasphere.exchangis.job.server.service.impl.DefaultJobExecuteService; -import com.webank.wedatasphere.exchangis.job.server.service.impl.ProjectImportServerImpl; import com.webank.wedatasphere.exchangis.job.server.utils.JobAuthorityUtils; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; -import com.webank.wedatasphere.exchangis.project.server.mapper.ProjectMapper; import org.apache.commons.lang.StringUtils; import org.apache.linkis.server.BDPJettyServerHelper; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.validation.BindingResult; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; @@ -32,8 +26,6 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.validation.groups.Default; -import javax.ws.rs.core.Context; -import java.rmi.ServerException; import java.util.Map; import java.util.Objects; @@ -57,15 +49,6 @@ public class ExchangisJobDssAppConnRestfulApi { @Resource private DefaultJobExecuteService executeService; - @Resource - private ProjectImportServerImpl projectImportServer; - - @Resource - private IProjectCopyService projectCopyService; - - @Autowired - private ProjectMapper projectMapper; - /** * Create job * @param request http request @@ -213,9 +196,6 @@ public Message executeJob(@PathVariable("id") Long id, HttpServletRequest reques jobInfo.setId(jobVo.getId()); LOG.info("jobInfo: name{},executerUser{},createUser{},id{}",jobInfo.getName(),jobInfo.getExecuteUser(),jobInfo.getCreateUser(),jobInfo.getId()); LOG.info("loginUser: {}, jobVo:{}",loginUser,jobVo); - //find project info - ExchangisProject project = projectMapper.getDetailById(jobVo.getProjectId()); - LOG.info("project: {}, getProjectId:{}",project,jobVo.getProjectId()); //find project user authority /*if (!hasAuthority(submitUser, jobVo)){ return Message.error("You have no permission to execute job (没有执行DSS任务权限)"); @@ -254,71 +234,4 @@ public Message executeJob(@PathVariable("id") Long id, HttpServletRequest reques AuditLogUtils.printLog(oringinUser, loginUser, TargetTypeEnum.JOB, id.toString(), "Execute task is: " + jobInfo.getName(), OperateTypeEnum.EXECUTE, request); return result; } - - @RequestMapping( value = "/import", method = RequestMethod.POST) - public Message importJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException{ - - Message response = null; - String userName = UserUtils.getLoginUser(request); - String oringinUser = SecurityFilter.getLoginUsername(request); - try { - LOG.info("param: {}", params); - /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { - return Message.error("You have no permission to import (没有导入权限)"); - }*/ - response = projectImportServer.importProject(request, params); - LOG.info("import job success"); - } catch (ExchangisJobServerException e){ - String message = "Fail import job [ id: " + params + "] (导入任务失败)"; - LOG.error(message, e); - response = Message.error(message); - } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.IMPORT, request); - return response; - - } - - @RequestMapping( value = "/export", method = RequestMethod.POST) - public Message exportJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { - String userName = UserUtils.getLoginUser(request); - - LOG.info("export function params: {}", params); - String oringinUser = SecurityFilter.getLoginUsername(request); - Message response = null; - try { - /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { - return Message.error("You have no permission to export (没有导出权限)"); - }*/ - response = jobInfoService.exportProject(params, userName, request); - LOG.info("export job success"); - } catch (Exception e){ - String message = "Fail Export job [ id: " + params + "] (导出任务失败)"; - LOG.error(message, e); - response = Message.error(message); - } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.EXPORT, request); - return response; - } - - @RequestMapping( value = "/copy", method = RequestMethod.POST) - public Message copy(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException { - String userName = UserUtils.getLoginUser(request); - String oringinUser = SecurityFilter.getLoginUsername(request); - - LOG.info("copy function params: {}", params); - Message response = null; - try { - response = projectCopyService.copy(params, userName, request); - LOG.info("copy node success"); - } catch (Exception e){ - String message = "Fail Copy project [ id: " + params + "] (导出任务失败)"; - LOG.error(message, e); - response = Message.error(message); - } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Copy parameter is: " + params.toString(), OperateTypeEnum.COPY, request); - return response; - - //return jobInfoService.exportProject(params, userName, request); - - } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectExportService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectExportService.java deleted file mode 100644 index feea880c0..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectExportService.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.service; - -import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; - -import javax.servlet.http.HttpServletRequest; -import java.util.Map; -import java.util.Set; - -/** - * @author tikazhang - * @Date 2022/3/15 9:30 - */ -public interface IProjectExportService { - String exportProject(Map params, String username, HttpServletRequest request) throws Exception; - - ExportedProject export(Long projectId, Map> moduleIdsMap, boolean partial); - - Map> getModuleIdsMap(Map params); - - Long getProjectId(Map> moduleIdsMap); -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java index 748d9e0bc..aee170717 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java @@ -2,21 +2,14 @@ import com.webank.wedatasphere.exchangis.common.pager.PageResult; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; -import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import org.apache.linkis.server.Message; import javax.servlet.http.HttpServletRequest; import java.util.List; -import java.rmi.ServerException; -import java.util.Map; -import java.util.Set; - - /** * The interface Exchangis job service. * @@ -106,18 +99,6 @@ public interface JobInfoService { */ ExchangisJobVo updateJobContent(ExchangisJobVo jobVo) throws ExchangisJobServerException, ExchangisDataSourceException; - /** - * Export exchangis job to BML. - * - * @param username params - * @return - */ - Message exportProject(Map params, String username, HttpServletRequest request) throws ExchangisJobServerException, ServerException; - - ExportedProject export(Long projectId, Map> moduleIdsMap, boolean partial, HttpServletRequest request) throws ExchangisJobServerException; - - Map> getModuleIdsMap(Map params); - /** * Copy job exchangis job basic info. * diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java index f2114c340..3acfbf5e1 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java @@ -6,8 +6,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import com.webank.wedatasphere.exchangis.common.pager.PageResult; import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobDsBind; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; @@ -16,24 +14,14 @@ import com.webank.wedatasphere.exchangis.datasource.core.vo.ExchangisJobInfoContent; import com.webank.wedatasphere.exchangis.datasource.service.ExchangisDataSourceService; import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; -import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; import com.webank.wedatasphere.exchangis.job.server.mapper.ExchangisJobEntityDao; -import com.webank.wedatasphere.exchangis.job.server.restful.external.ModuleEnum; import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; -import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; -import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.apache.linkis.bml.client.BmlClient; -import org.apache.linkis.bml.client.BmlClientFactory; -import org.apache.linkis.bml.protocol.BmlUploadResponse; import org.apache.linkis.common.utils.JsonUtils; import org.apache.linkis.manager.label.utils.LabelUtils; -import org.apache.linkis.server.BDPJettyServerHelper; -import org.apache.linkis.server.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -42,9 +30,6 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; -import java.rmi.ServerException; import java.util.*; import java.util.stream.Collectors; @@ -56,12 +41,6 @@ public class DefaultJobInfoService implements JobInfoService { private static final Logger LOG = LoggerFactory.getLogger(DefaultJobInfoService.class); - /** - * Project service - */ - @Resource - private ProjectService projectService; - @Autowired private ExchangisJobDsBindServiceImpl exchangisJobDsBindService; @@ -304,161 +283,6 @@ public ExchangisJobVo updateJobContent(ExchangisJobVo jobVo) throws ExchangisJob return jobVo; } - @Override - public Message exportProject(Map params, String userName, HttpServletRequest request) throws ExchangisJobServerException, ServerException { - ExportedProject exportedProject = null; - Long projectId = Long.parseLong(params.get("projectId").toString()); - Boolean partial = (Boolean) params.get("partial"); - Map> moduleIdsMap = getModuleIdsMap(params); - - LOG.info("export project, user: {}, project: {}, partial:{}", userName, projectId, partial); - exportedProject = export(projectId, moduleIdsMap, partial, request); - String exported = null; - try { - exported = BDPJettyServerHelper.jacksonJson().writeValueAsString(exportedProject); - } catch (JsonProcessingException e) { - LOG.error("Occur error while tranform class", e.getMessage()); - } - - LOG.info("projectName: {}, exported:{}", exportedProject.getName(), exported); - BmlClient bmlClient = BmlClientFactory.createBmlClient(userName); - BmlUploadResponse bmlUploadResponse = bmlClient.uploadShareResource(userName, exportedProject.getName(), - "exchangis_exported_" + UUID.randomUUID(), new ByteArrayInputStream(exported.getBytes(StandardCharsets.UTF_8))); - - if (bmlUploadResponse == null || !bmlUploadResponse.isSuccess()) { - throw new ServerException("cannot upload exported data to BML"); - } - - LOG.info("{} is exporting the project, uploaded to BML the resourceID is {} and the version is {}", - userName, bmlUploadResponse.resourceId(), bmlUploadResponse.version()); - - Message message = Message.ok("export job") - .data("resourceId", bmlUploadResponse.resourceId()) - .data("version", bmlUploadResponse.version()); - return message; - } - - @Override - public ExportedProject export(Long projectId, Map> moduleIdsMap, boolean partial, HttpServletRequest request) throws ExchangisJobServerException { - ExportedProject exportedProject = new ExportedProject(); - ExchangisProjectInfo project = projectService.getProjectDetailById(projectId); - - LOG.info("execute export method! export project is {}.", project.getName()); - exportedProject.setName(project.getName()); - - setSqoop(projectId, moduleIdsMap, partial, exportedProject, request); - - setDatax(projectId, moduleIdsMap, partial, exportedProject, request); - - return exportedProject; - } - - private void setSqoop(Long projectId, Map> moduleIdsMap, boolean partial, ExportedProject exportedProject, HttpServletRequest request) throws ExchangisJobServerException { - List sqoops = new ArrayList<>(); - LOG.info("Request: {}", request); - if (partial) { - Set longs = moduleIdsMap.get(ModuleEnum.SQOOP_IDS.getName()); - if (longs.size() > 0) { - for (Long id : longs) { - LOG.info("id: {}", id); - ExchangisJobVo job = jobInfoService.getJob(id, false); - - String sqoopStr = null; - try { - sqoopStr = BDPJettyServerHelper.jacksonJson().writeValueAsString(job); - } catch (JsonProcessingException e) { - LOG.error("Occur error while tranform class", e.getMessage()); - } - - LOG.info("sqoopStr:{}", sqoopStr); - LOG.info("ExchangisJobVo sqoop: {}", job.getContent()); - LOG.info("getCreateTime: {}", job.getId()); - LOG.info("executorUser: {}", job.getExecuteUser()); - sqoops.add(job); - } - exportedProject.setSqoops(sqoops); - } - - } else { - LOG.info("Through request {} and projectId {} get Sqoopjob", request, projectId); - sqoops = jobInfoService.getSubJobList(request, projectId); - exportedProject.setSqoops(sqoops); - //exportedProject.setSqoops(jobInfoService.getByProject(request, projectId)); - } - LOG.info("exporting project, export sqoopJob: {}", exportedProject); - } - - private void setDatax(Long projectId, Map> moduleIdsMap, boolean partial, ExportedProject exportedProject, HttpServletRequest request) throws ExchangisJobServerException { - List dataxs = new ArrayList<>(); - LOG.info("Request: {}", request); - if (partial) { - Set longs = moduleIdsMap.get(ModuleEnum.DATAX_IDS.getName()); - if (longs.size() > 0) { - for (Long id : longs) { - LOG.info("id: {}", id); - ExchangisJobVo job = jobInfoService.getJob(id, false); - - String dataxStr = null; - try { - dataxStr = BDPJettyServerHelper.jacksonJson().writeValueAsString(job); - } catch (JsonProcessingException e) { - LOG.error("Occur error while tranform class", e.getMessage()); - } - - LOG.info("dataxStr:{}", dataxStr); - LOG.info("ExchangisJobVo sqoop: {}", job.getContent()); - LOG.info("getCreateTime: {}", job.getId()); - LOG.info("executorUser: {}", job.getExecuteUser()); - dataxs.add(job); - } - exportedProject.setDataxes(dataxs); - } - - } else { - LOG.info("Through request {} and projectId {} get Dataxjob", request, projectId); - dataxs = jobInfoService.getSubJobList(request, projectId); - exportedProject.setSqoops(dataxs); - } - LOG.info("exporting project, export dataxJob: {}", exportedProject); - - } - - /** - * 获取需要导出对象集合 - * - * @param params - * @return - */ - @Override - public Map> getModuleIdsMap(Map params) { - - Map> map = Maps.newHashMap(); - String sqoopIdsStr = null; - if(params.get("sqoopIds") != null) { - sqoopIdsStr = params.get("sqoopIds").toString(); - } - String dataxIdsStr = null; - if(params.get("dataXIds") != null) { - dataxIdsStr = params.get("dataXIds").toString(); - } - - Set sqoopIds = Sets.newHashSet(); - Set dataxIds = Sets.newHashSet(); - - if (StringUtils.isNotEmpty(sqoopIdsStr)) { - sqoopIds = Arrays.stream(StringUtils.split(sqoopIdsStr, ",")) - .map(Long::parseLong).collect(Collectors.toSet()); - } - if (StringUtils.isNotEmpty(dataxIdsStr)) { - dataxIds = Arrays.stream(StringUtils.split(dataxIdsStr, ",")) - .map(Long::parseLong).collect(Collectors.toSet()); - } - map.put("sqoopIds", sqoopIds); - map.put("dataXIds", dataxIds); - LOG.info("The objects to be exported are: {}", map); - return map; - } - @Override @Transactional(rollbackFor = Exception.class) public ExchangisJobVo copyJob(ExchangisJobVo jobVo) { diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/JobAuthorityUtils.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/JobAuthorityUtils.java index a39d5dac0..29fd31713 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/JobAuthorityUtils.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/JobAuthorityUtils.java @@ -7,8 +7,8 @@ import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; import com.webank.wedatasphere.exchangis.job.server.mapper.ExchangisJobEntityDao; import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedJobDao; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectUser; -import com.webank.wedatasphere.exchangis.project.server.mapper.ProjectUserMapper; +import com.webank.wedatasphere.exchangis.project.entity.domain.ExchangisProjectUser; +import com.webank.wedatasphere.exchangis.project.provider.mapper.ProjectUserMapper; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; diff --git a/exchangis-job/exchangis-job-service/pom.xml b/exchangis-job/exchangis-job-service/pom.xml new file mode 100644 index 000000000..c63896b36 --- /dev/null +++ b/exchangis-job/exchangis-job-service/pom.xml @@ -0,0 +1,33 @@ + + + + exchangis + com.webank.wedatasphere.exchangis + 1.1.1 + ../../pom.xml + + 4.0.0 + + exchangis-job-service + + + 8 + 8 + + + + + com.webank.wedatasphere.exchangis + exchangis-job-common + 1.1.1 + + + com.webank.wedatasphere.exchangis + exchangis-job-launcher + 1.1.1 + + + + \ No newline at end of file diff --git a/exchangis-project/exchangis-project-entity/pom.xml b/exchangis-project/exchangis-project-entity/pom.xml new file mode 100644 index 000000000..198497921 --- /dev/null +++ b/exchangis-project/exchangis-project-entity/pom.xml @@ -0,0 +1,32 @@ + + + + exchangis-project + com.webank.wedatasphere.exchangis + 1.1.2 + + 4.0.0 + + exchangis-project-entity + + + 8 + 8 + + + + + com.webank.wedatasphere.exchangis + exchangis-dao + 1.1.2 + + + org.apache.commons + commons-math3 + 3.6.1 + + + + \ No newline at end of file diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProjectUser.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ExchangisProjectUser.java similarity index 95% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProjectUser.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ExchangisProjectUser.java index ff460da4f..195741fa6 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProjectUser.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ExchangisProjectUser.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.project.server.entity; +package com.webank.wedatasphere.exchangis.project.entity.domain; import java.util.Date; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/OperationType.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/OperationType.java similarity index 86% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/OperationType.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/OperationType.java index fcb958a6a..ab8f2ccf4 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/OperationType.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/OperationType.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.project.server.domain; +package com.webank.wedatasphere.exchangis.project.entity.domain; /** * @author jefftlin diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/ProjectPageQuery.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ProjectPageQuery.java similarity index 83% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/ProjectPageQuery.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ProjectPageQuery.java index 9a4a012f3..db42b536c 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/ProjectPageQuery.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ProjectPageQuery.java @@ -1,12 +1,12 @@ -package com.webank.wedatasphere.exchangis.project.server.domain; +package com.webank.wedatasphere.exchangis.project.entity.domain; import com.webank.wedatasphere.exchangis.common.pager.PageQuery; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; /** * For querying page */ public class ProjectPageQuery extends PageQuery { + /** * Project name */ diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProject.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/entity/ExchangisProject.java similarity index 98% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProject.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/entity/ExchangisProject.java index b0fdc0a6b..286c2ceb1 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProject.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/entity/ExchangisProject.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.project.server.entity; +package com.webank.wedatasphere.exchangis.project.entity.entity; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectAppVo.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectAppVo.java similarity index 97% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectAppVo.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectAppVo.java index f3ec21097..d6ca525c5 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectAppVo.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectAppVo.java @@ -1,8 +1,8 @@ -package com.webank.wedatasphere.exchangis.project.server.vo; +package com.webank.wedatasphere.exchangis.project.entity.vo; import com.fasterxml.jackson.annotation.JsonAlias; import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.entity.entity.ExchangisProject; import org.hibernate.validator.constraints.NotBlank; import javax.validation.constraints.NotNull; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectInfo.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectInfo.java similarity index 96% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectInfo.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectInfo.java index fae790d63..dc356715b 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectInfo.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectInfo.java @@ -1,9 +1,8 @@ -package com.webank.wedatasphere.exchangis.project.server.vo; +package com.webank.wedatasphere.exchangis.project.entity.vo; import com.fasterxml.jackson.annotation.JsonAlias; -import com.fasterxml.jackson.annotation.JsonInclude; import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.entity.entity.ExchangisProject; import org.hibernate.validator.constraints.NotBlank; import javax.validation.constraints.NotNull; @@ -18,7 +17,7 @@ * TODO @JsonInclude(JsonInclude.Include.NON_EMPTY) */ //@JsonInclude(JsonInclude.Include.NON_EMPTY) -public class ExchangisProjectInfo { +public class ExchangisProjectInfo { /** * ID */ diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectUserVo.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectUserVo.java similarity index 90% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectUserVo.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectUserVo.java index b959a5dc3..fef892115 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectUserVo.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectUserVo.java @@ -1,9 +1,8 @@ -package com.webank.wedatasphere.exchangis.project.server.vo; +package com.webank.wedatasphere.exchangis.project.entity.vo; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectUser; +import com.webank.wedatasphere.exchangis.project.entity.domain.ExchangisProjectUser; import javax.validation.constraints.NotNull; -import java.util.Date; /** * @author tikazhang diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ProjectQueryVo.java similarity index 87% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ProjectQueryVo.java index 51d5a4a99..42498c95f 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ProjectQueryVo.java @@ -1,7 +1,7 @@ -package com.webank.wedatasphere.exchangis.project.server.vo; +package com.webank.wedatasphere.exchangis.project.entity.vo; import com.fasterxml.jackson.annotation.JsonInclude; -import com.webank.wedatasphere.exchangis.project.server.domain.ProjectPageQuery; +import com.webank.wedatasphere.exchangis.project.entity.domain.ProjectPageQuery; import java.util.Map; import java.util.Optional; @@ -15,6 +15,7 @@ public class ProjectQueryVo extends ProjectPageQuery { public ProjectQueryVo(){ } + private Map labels; public Map getLabels() { diff --git a/exchangis-project/exchangis-project-provider/pom.xml b/exchangis-project/exchangis-project-provider/pom.xml new file mode 100644 index 000000000..0a94a8840 --- /dev/null +++ b/exchangis-project/exchangis-project-provider/pom.xml @@ -0,0 +1,27 @@ + + + + exchangis-project + com.webank.wedatasphere.exchangis + 1.1.2 + + 4.0.0 + + exchangis-project-provider + + + 8 + 8 + + + + + com.webank.wedatasphere.exchangis + exchangis-project-entity + 1.1.2 + + + + \ No newline at end of file diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectMapper.java b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectMapper.java similarity index 90% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectMapper.java rename to exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectMapper.java index cbdc53bdb..4c7b7f41a 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectMapper.java +++ b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectMapper.java @@ -1,7 +1,7 @@ -package com.webank.wedatasphere.exchangis.project.server.mapper; +package com.webank.wedatasphere.exchangis.project.provider.mapper; -import com.webank.wedatasphere.exchangis.project.server.domain.ProjectPageQuery; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.entity.domain.ProjectPageQuery; +import com.webank.wedatasphere.exchangis.project.entity.entity.ExchangisProject; import org.apache.ibatis.annotations.Param; import java.util.List; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectUserMapper.java b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectUserMapper.java similarity index 85% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectUserMapper.java rename to exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectUserMapper.java index 8a5a09c85..33cac2930 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectUserMapper.java +++ b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectUserMapper.java @@ -1,6 +1,6 @@ -package com.webank.wedatasphere.exchangis.project.server.mapper; +package com.webank.wedatasphere.exchangis.project.provider.mapper; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectUser; +import com.webank.wedatasphere.exchangis.project.entity.domain.ExchangisProjectUser; import org.apache.ibatis.annotations.Param; import java.util.List; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectMapper.xml similarity index 97% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml rename to exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectMapper.xml index 5fc2ca755..70225fb67 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml +++ b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectMapper.xml @@ -1,9 +1,9 @@ - + - + diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectUserMapper.xml b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectUserMapper.xml similarity index 92% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectUserMapper.xml rename to exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectUserMapper.xml index 6335cd8fe..9a8e2d257 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectUserMapper.xml +++ b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectUserMapper.xml @@ -1,9 +1,9 @@ - + - + diff --git a/exchangis-project/exchangis-project-server/pom.xml b/exchangis-project/exchangis-project-server/pom.xml index c0e02e2a6..0853557d1 100644 --- a/exchangis-project/exchangis-project-server/pom.xml +++ b/exchangis-project/exchangis-project-server/pom.xml @@ -18,14 +18,14 @@ - org.apache.linkis - linkis-mybatis - ${linkis.version} + com.webank.wedatasphere.exchangis + exchangis-project-provider + 1.1.2 - org.apache.commons - commons-math3 - 3.6.1 + com.webank.wedatasphere.exchangis + exchangis-job-server + 1.1.2 com.webank.wedatasphere.exchangis diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java index c6fff1573..57b54fb3b 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java @@ -7,16 +7,19 @@ import com.webank.wedatasphere.exchangis.common.enums.TargetTypeEnum; import com.webank.wedatasphere.exchangis.common.pager.PageResult; import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; -import com.webank.wedatasphere.exchangis.project.server.domain.OperationType; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectUser; +import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; +import com.webank.wedatasphere.exchangis.project.entity.domain.OperationType; +import com.webank.wedatasphere.exchangis.project.entity.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.entity.domain.ExchangisProjectUser; import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; import com.webank.wedatasphere.exchangis.project.server.utils.ProjectAuthorityUtils; import com.webank.wedatasphere.exchangis.project.server.utils.ExchangisProjectConfiguration; import com.webank.wedatasphere.exchangis.project.server.utils.ExchangisProjectRestfulUtils; -import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; -import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectUserVo; -import com.webank.wedatasphere.exchangis.project.server.vo.ProjectQueryVo; +import com.webank.wedatasphere.exchangis.project.entity.vo.ExchangisProjectInfo; +import com.webank.wedatasphere.exchangis.project.entity.vo.ExchangisProjectUserVo; +import com.webank.wedatasphere.exchangis.project.entity.vo.ProjectQueryVo; import org.apache.commons.lang3.StringUtils; import org.apache.commons.math3.util.Pair; import org.apache.linkis.common.utils.JsonUtils; @@ -49,6 +52,12 @@ public class ExchangisProjectRestfulApi { @Resource private ProjectService projectService; + /** + * JobInfo service + */ + @Resource + private JobInfoService jobInfoService; + /** * Project query * @param request http request @@ -208,6 +217,7 @@ public Message updateProject(@Validated({UpdateGroup.class, Default.class}) @Req .equalsIgnoreCase(domain)){ return Message.error("Cannot update the outer project (无法更新来自 " + domain + " 的外部项目)"); } + LOG.info("UpdateProject vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); projectService.updateProject(projectVo, username); AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, projectVo.getId(), "Project name is: " + projectVo.getName(), OperateTypeEnum.UPDATE, request); @@ -236,7 +246,16 @@ public Message deleteProject(HttpServletRequest request, @PathVariable("id") Lon try { ExchangisProjectInfo projectInfo = projectService.getProjectDetailById(id); if (!ProjectAuthorityUtils.hasProjectAuthority(username, projectInfo, OperationType.PROJECT_ALTER)) { - return Message.error("You have no permission to delete (删除项目失败)"); + return Message.error("You have no permission to delete (没有权限删除项目)"); + } + + // 校验是否有任务 + ExchangisJobQueryVo queryVo = new ExchangisJobQueryVo( + id, null, null, null, null); + PageResult exchangisJobVoPageResult = jobInfoService.queryJobList(queryVo); + if (Objects.nonNull(exchangisJobVoPageResult) && Objects.nonNull(exchangisJobVoPageResult.getList()) + && exchangisJobVoPageResult.getList().size() > 0) { + return Message.error("Jobs already exist under this project and the project cannot be deleted (该项目下已存在子任务，无法删除)"); } String domain = projectInfo.getDomain(); diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisJobDssAppConnRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisJobDssAppConnRestfulApi.java new file mode 100644 index 000000000..b2d41933d --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisJobDssAppConnRestfulApi.java @@ -0,0 +1,109 @@ +package com.webank.wedatasphere.exchangis.project.server.restful.external; + +import com.webank.wedatasphere.exchangis.common.AuditLogUtils; +import com.webank.wedatasphere.exchangis.common.UserUtils; +import com.webank.wedatasphere.exchangis.common.enums.OperateTypeEnum; +import com.webank.wedatasphere.exchangis.common.enums.TargetTypeEnum; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.project.provider.mapper.ProjectMapper; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectCopyService; +import com.webank.wedatasphere.exchangis.project.server.service.impl.ProjectExportServiceImpl; +import com.webank.wedatasphere.exchangis.project.server.service.impl.ProjectImportServerImpl; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.Context; +import java.rmi.ServerException; +import java.util.Map; + +/** + * Define to support the app conn, in order to distinguish from the inner api + */ +@RestController +@RequestMapping(value = "/dss/exchangis/main/appJob", produces = {"application/json;charset=utf-8"}) +public class ExchangisJobDssAppConnRestfulApi { + + private static final Logger LOG = LoggerFactory.getLogger(ExchangisJobDssAppConnRestfulApi.class); + + @Resource + private ProjectImportServerImpl projectImportServer; + + @Resource + private ProjectExportServiceImpl projectExportService; + + @Resource + private ProjectCopyService projectCopyService; + + @Autowired + private ProjectMapper projectMapper; + + @RequestMapping(value = "/import", method = RequestMethod.POST) + public Message importJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { + + Message response = null; + String userName = UserUtils.getLoginUser(request); + String oringinUser = SecurityFilter.getLoginUsername(request); + try { + LOG.info("param: {}", params); + /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { + return Message.error("You have no permission to import (没有导入权限)"); + }*/ + response = projectImportServer.importProject(request, params); + LOG.info("import job success"); + } catch (ExchangisJobServerException e) { + String message = "Fail import job [ id: " + params + "] (导入任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.IMPORT, request); + return response; + + } + + @RequestMapping(value = "/export", method = RequestMethod.POST) + public Message exportJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { + String userName = UserUtils.getLoginUser(request); + + LOG.info("export function params: {}", params); + String oringinUser = SecurityFilter.getLoginUsername(request); + Message response = null; + try { + /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { + return Message.error("You have no permission to export (没有导出权限)"); + }*/ + response = projectExportService.exportProject(params, userName, request); + LOG.info("export job success"); + } catch (Exception e) { + String message = "Fail Export job [ id: " + params + "] (导出任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.EXPORT, request); + return response; + } + + @RequestMapping(value = "/copy", method = RequestMethod.POST) + public Message copy(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException { + String userName = UserUtils.getLoginUser(request); + String oringinUser = SecurityFilter.getLoginUsername(request); + + LOG.info("copy function params: {}", params); + Message response = null; + try { + response = projectCopyService.copy(params, userName, request); + LOG.info("copy node success"); + } catch (Exception e) { + String message = "Fail Copy project [ id: " + params + "] (导出任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Copy parameter is: " + params.toString(), OperateTypeEnum.COPY, request); + return response; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectCopyService.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectCopyService.java similarity index 83% rename from exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectCopyService.java rename to exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectCopyService.java index e5d5f724d..9260d02bf 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectCopyService.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectCopyService.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.job.server.service; +package com.webank.wedatasphere.exchangis.project.server.service; import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; @@ -11,7 +11,7 @@ * @author tikazhang * @Date 2022/4/24 21:11 */ -public interface IProjectCopyService { +public interface ProjectCopyService { /** * Copy node diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectExportService.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectExportService.java new file mode 100644 index 000000000..81fddc9eb --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectExportService.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.exchangis.project.server.service; + +import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import org.apache.linkis.server.Message; + +import javax.servlet.http.HttpServletRequest; +import java.rmi.ServerException; +import java.util.Map; +import java.util.Set; + +/** + * @author tikazhang + * @Date 2022/3/15 9:30 + */ +public interface ProjectExportService { + + /** + * Export exchangis job to BML. + * + * @param username params + * @return + */ + Message exportProject(Map params, String username, HttpServletRequest request) throws ExchangisJobServerException, ServerException; + + ExportedProject export(Long projectId, Map> moduleIdsMap, boolean partial, HttpServletRequest request) throws ExchangisJobServerException; + + Map> getModuleIdsMap(Map params); + +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectImportService.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectImportService.java similarity index 76% rename from exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectImportService.java rename to exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectImportService.java index efb451acd..34b1a2219 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectImportService.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectImportService.java @@ -1,8 +1,7 @@ -package com.webank.wedatasphere.exchangis.job.server.service; +package com.webank.wedatasphere.exchangis.project.server.service; import com.webank.wedatasphere.exchangis.job.server.dto.IdCatalog; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.project.server.exception.ExchangisProjectErrorException; import org.apache.linkis.server.Message; import javax.servlet.http.HttpServletRequest; @@ -13,7 +12,7 @@ * @author tikazhang * @Date 2022/3/15 10:01 */ -public interface IProjectImportService { +public interface ProjectImportService { Message importProject(HttpServletRequest req, Map params) throws ExchangisJobServerException, ServerException; IdCatalog importOpt(String projectJson, Long projectId, String versionSuffix, String userName, String importType) throws ExchangisJobServerException; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectService.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectService.java index 54136b26c..9c6556a34 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectService.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectService.java @@ -3,10 +3,10 @@ import com.webank.wedatasphere.exchangis.common.pager.PageResult; import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectUser; -import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; -import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectUserVo; -import com.webank.wedatasphere.exchangis.project.server.vo.ProjectQueryVo; +import com.webank.wedatasphere.exchangis.project.entity.domain.ExchangisProjectUser; +import com.webank.wedatasphere.exchangis.project.entity.vo.ExchangisProjectInfo; +import com.webank.wedatasphere.exchangis.project.entity.vo.ExchangisProjectUserVo; +import com.webank.wedatasphere.exchangis.project.entity.vo.ProjectQueryVo; /** * Project query diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectCopyServiceImpl.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectCopyServiceImpl.java similarity index 85% rename from exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectCopyServiceImpl.java rename to exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectCopyServiceImpl.java index 13ee9f1d4..58bb7e94a 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectCopyServiceImpl.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectCopyServiceImpl.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.job.server.service.impl; +package com.webank.wedatasphere.exchangis.project.server.service.impl; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.Lists; @@ -9,8 +9,8 @@ import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; import com.webank.wedatasphere.exchangis.job.server.mapper.ExchangisJobEntityDao; import com.webank.wedatasphere.exchangis.job.server.restful.external.ModuleEnum; -import com.webank.wedatasphere.exchangis.job.server.service.IProjectCopyService; -import com.webank.wedatasphere.exchangis.job.server.service.IProjectImportService; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectCopyService; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectImportService; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; import org.apache.commons.lang.StringUtils; import org.apache.linkis.server.BDPJettyServerHelper; @@ -29,14 +29,14 @@ * @Date 2022/4/24 21:15 */ @Service -public class ProjectCopyServiceImpl implements IProjectCopyService { +public class ProjectCopyServiceImpl implements ProjectCopyService { private static final Logger LOG = LoggerFactory.getLogger(ProjectCopyServiceImpl.class); @Autowired - private IProjectImportService projectImportService; + private ProjectImportService projectImportService; @Autowired - private DefaultJobInfoService defaultJobInfoService; + private ProjectExportServiceImpl projectExportService; @Autowired private ProjectImportServerImpl projectImportServer; @@ -49,7 +49,7 @@ public Message copy(Map params, String userName, HttpServletRequ LOG.info("begin to copy in project params is {}", params); //Long projectId = Long.parseLong(params.get("projectId").toString()); Boolean partial = (Boolean) params.get("partial"); - Map> moduleIdsMap = defaultJobInfoService.getModuleIdsMap(params); + Map> moduleIdsMap = projectExportService.getModuleIdsMap(params); Set longs = moduleIdsMap.get(Objects.isNull(params.get("dataXIds")) ? ModuleEnum.SQOOP_IDS.getName() : ModuleEnum.DATAX_IDS.getName()); List list1 = new ArrayList(longs); @@ -64,7 +64,7 @@ public Message copy(Map params, String userName, HttpServletRequ } String contextIdStr = (String) params.get("contextID"); - ExportedProject exportedProject = defaultJobInfoService.export(projectId, moduleIdsMap, partial, request); + ExportedProject exportedProject = projectExportService.export(projectId, moduleIdsMap, partial, request); copySqoop(moduleIdsMap, exportedProject); diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectExportServiceImpl.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectExportServiceImpl.java new file mode 100644 index 000000000..3fcd0f1d6 --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectExportServiceImpl.java @@ -0,0 +1,202 @@ +package com.webank.wedatasphere.exchangis.project.server.service.impl; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.job.server.restful.external.ModuleEnum; +import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectExportService; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; +import com.webank.wedatasphere.exchangis.project.entity.vo.ExchangisProjectInfo; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.bml.client.BmlClient; +import org.apache.linkis.bml.client.BmlClientFactory; +import org.apache.linkis.bml.protocol.BmlUploadResponse; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.rmi.ServerException; +import java.util.*; +import java.util.stream.Collectors; + +/** + * @author jefftlin + * @date 2023/7/13 + */ +public class ProjectExportServiceImpl implements ProjectExportService { + + private static final Logger LOG = LoggerFactory.getLogger(ProjectExportServiceImpl.class); + + /** + * Project service + */ + @Resource + private ProjectService projectService; + + @Resource + private JobInfoService jobInfoService; + + @Override + public Message exportProject(Map params, String userName, HttpServletRequest request) throws ExchangisJobServerException, ServerException { + ExportedProject exportedProject = null; + Long projectId = Long.parseLong(params.get("projectId").toString()); + Boolean partial = (Boolean) params.get("partial"); + Map> moduleIdsMap = getModuleIdsMap(params); + + LOG.info("export project, user: {}, project: {}, partial:{}", userName, projectId, partial); + exportedProject = export(projectId, moduleIdsMap, partial, request); + String exported = null; + try { + exported = BDPJettyServerHelper.jacksonJson().writeValueAsString(exportedProject); + } catch (JsonProcessingException e) { + LOG.error("Occur error while tranform class", e.getMessage()); + } + + LOG.info("projectName: {}, exported:{}", exportedProject.getName(), exported); + BmlClient bmlClient = BmlClientFactory.createBmlClient(userName); + BmlUploadResponse bmlUploadResponse = bmlClient.uploadShareResource(userName, exportedProject.getName(), + "exchangis_exported_" + UUID.randomUUID(), new ByteArrayInputStream(exported.getBytes(StandardCharsets.UTF_8))); + + if (bmlUploadResponse == null || !bmlUploadResponse.isSuccess()) { + throw new ServerException("cannot upload exported data to BML"); + } + + LOG.info("{} is exporting the project, uploaded to BML the resourceID is {} and the version is {}", + userName, bmlUploadResponse.resourceId(), bmlUploadResponse.version()); + + Message message = Message.ok("export job") + .data("resourceId", bmlUploadResponse.resourceId()) + .data("version", bmlUploadResponse.version()); + return message; + } + + @Override + public ExportedProject export(Long projectId, Map> moduleIdsMap, boolean partial, HttpServletRequest request) throws ExchangisJobServerException { + ExportedProject exportedProject = new ExportedProject(); + ExchangisProjectInfo project = projectService.getProjectDetailById(projectId); + + LOG.info("execute export method! export project is {}.", project.getName()); + exportedProject.setName(project.getName()); + + setSqoop(projectId, moduleIdsMap, partial, exportedProject, request); + + setDatax(projectId, moduleIdsMap, partial, exportedProject, request); + + return exportedProject; + } + + private void setSqoop(Long projectId, Map> moduleIdsMap, boolean partial, ExportedProject exportedProject, HttpServletRequest request) throws ExchangisJobServerException { + List sqoops = new ArrayList<>(); + LOG.info("Request: {}", request); + if (partial) { + Set longs = moduleIdsMap.get(ModuleEnum.SQOOP_IDS.getName()); + if (longs.size() > 0) { + for (Long id : longs) { + LOG.info("id: {}", id); + ExchangisJobVo job = jobInfoService.getJob(id, false); + + String sqoopStr = null; + try { + sqoopStr = BDPJettyServerHelper.jacksonJson().writeValueAsString(job); + } catch (JsonProcessingException e) { + LOG.error("Occur error while tranform class", e.getMessage()); + } + + LOG.info("sqoopStr:{}", sqoopStr); + LOG.info("ExchangisJobVo sqoop: {}", job.getContent()); + LOG.info("getCreateTime: {}", job.getId()); + LOG.info("executorUser: {}", job.getExecuteUser()); + sqoops.add(job); + } + exportedProject.setSqoops(sqoops); + } + + } else { + LOG.info("Through request {} and projectId {} get Sqoopjob", request, projectId); + sqoops = jobInfoService.getSubJobList(request, projectId); + exportedProject.setSqoops(sqoops); + //exportedProject.setSqoops(jobInfoService.getByProject(request, projectId)); + } + LOG.info("exporting project, export sqoopJob: {}", exportedProject); + } + + private void setDatax(Long projectId, Map> moduleIdsMap, boolean partial, ExportedProject exportedProject, HttpServletRequest request) throws ExchangisJobServerException { + List dataxs = new ArrayList<>(); + LOG.info("Request: {}", request); + if (partial) { + Set longs = moduleIdsMap.get(ModuleEnum.DATAX_IDS.getName()); + if (longs.size() > 0) { + for (Long id : longs) { + LOG.info("id: {}", id); + ExchangisJobVo job = jobInfoService.getJob(id, false); + + String dataxStr = null; + try { + dataxStr = BDPJettyServerHelper.jacksonJson().writeValueAsString(job); + } catch (JsonProcessingException e) { + LOG.error("Occur error while tranform class", e.getMessage()); + } + + LOG.info("dataxStr:{}", dataxStr); + LOG.info("ExchangisJobVo sqoop: {}", job.getContent()); + LOG.info("getCreateTime: {}", job.getId()); + LOG.info("executorUser: {}", job.getExecuteUser()); + dataxs.add(job); + } + exportedProject.setDataxes(dataxs); + } + + } else { + LOG.info("Through request {} and projectId {} get Dataxjob", request, projectId); + dataxs = jobInfoService.getSubJobList(request, projectId); + exportedProject.setSqoops(dataxs); + } + LOG.info("exporting project, export dataxJob: {}", exportedProject); + + } + + /** + * 获取需要导出对象集合 + * + * @param params + * @return + */ + @Override + public Map> getModuleIdsMap(Map params) { + + Map> map = Maps.newHashMap(); + String sqoopIdsStr = null; + if(params.get("sqoopIds") != null) { + sqoopIdsStr = params.get("sqoopIds").toString(); + } + String dataxIdsStr = null; + if(params.get("dataXIds") != null) { + dataxIdsStr = params.get("dataXIds").toString(); + } + + Set sqoopIds = Sets.newHashSet(); + Set dataxIds = Sets.newHashSet(); + + if (StringUtils.isNotEmpty(sqoopIdsStr)) { + sqoopIds = Arrays.stream(StringUtils.split(sqoopIdsStr, ",")) + .map(Long::parseLong).collect(Collectors.toSet()); + } + if (StringUtils.isNotEmpty(dataxIdsStr)) { + dataxIds = Arrays.stream(StringUtils.split(dataxIdsStr, ",")) + .map(Long::parseLong).collect(Collectors.toSet()); + } + map.put("sqoopIds", sqoopIds); + map.put("dataXIds", dataxIds); + LOG.info("The objects to be exported are: {}", map); + return map; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectImportServerImpl.java similarity index 95% rename from exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java rename to exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectImportServerImpl.java index 37ba84083..6ca3ab4cb 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectImportServerImpl.java @@ -1,15 +1,13 @@ -package com.webank.wedatasphere.exchangis.job.server.service.impl; +package com.webank.wedatasphere.exchangis.project.server.service.impl; import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; import com.webank.wedatasphere.exchangis.job.server.dto.IdCatalog; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.job.server.service.IProjectImportService; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectImportService; import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; -import com.webank.wedatasphere.exchangis.project.server.exception.ExchangisProjectErrorException; -import com.webank.wedatasphere.exchangis.project.server.mapper.ProjectMapper; -import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; +import com.webank.wedatasphere.exchangis.project.entity.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.provider.mapper.ProjectMapper; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.linkis.bml.client.BmlClient; @@ -40,7 +38,7 @@ */ @Service -public class ProjectImportServerImpl implements IProjectImportService { +public class ProjectImportServerImpl implements ProjectImportService { private static final Logger LOG = LoggerFactory.getLogger(ProjectImportServerImpl.class); diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectServiceImpl.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectServiceImpl.java index 92c2541d2..89f3e12e6 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectServiceImpl.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectServiceImpl.java @@ -10,14 +10,14 @@ import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectUser; -import com.webank.wedatasphere.exchangis.project.server.mapper.ProjectMapper; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; -import com.webank.wedatasphere.exchangis.project.server.mapper.ProjectUserMapper; +import com.webank.wedatasphere.exchangis.project.provider.mapper.ProjectMapper; +import com.webank.wedatasphere.exchangis.project.provider.mapper.ProjectUserMapper; import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; -import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; -import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectUserVo; -import com.webank.wedatasphere.exchangis.project.server.vo.ProjectQueryVo; +import com.webank.wedatasphere.exchangis.project.entity.vo.ExchangisProjectInfo; +import com.webank.wedatasphere.exchangis.project.entity.vo.ExchangisProjectUserVo; +import com.webank.wedatasphere.exchangis.project.entity.vo.ProjectQueryVo; +import com.webank.wedatasphere.exchangis.project.entity.domain.ExchangisProjectUser; +import com.webank.wedatasphere.exchangis.project.entity.entity.ExchangisProject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/utils/ProjectAuthorityUtils.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/utils/ProjectAuthorityUtils.java index 27236ca8c..6fd06bf8f 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/utils/ProjectAuthorityUtils.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/utils/ProjectAuthorityUtils.java @@ -1,9 +1,9 @@ package com.webank.wedatasphere.exchangis.project.server.utils; -import com.webank.wedatasphere.exchangis.project.server.domain.OperationType; +import com.webank.wedatasphere.exchangis.project.entity.domain.OperationType; import com.webank.wedatasphere.exchangis.project.server.exception.ExchangisProjectErrorException; import com.webank.wedatasphere.exchangis.project.server.exception.ExchangisProjectExceptionCode; -import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; +import com.webank.wedatasphere.exchangis.project.entity.vo.ExchangisProjectInfo; import org.apache.commons.lang3.StringUtils; import java.util.Arrays; diff --git a/exchangis-project/pom.xml b/exchangis-project/pom.xml index 4b7f2e1ac..99a3c3a17 100644 --- a/exchangis-project/pom.xml +++ b/exchangis-project/pom.xml @@ -13,6 +13,8 @@ pom exchangis-project-server + exchangis-project-provider + exchangis-project-entity From edf09f595c25ea9ee9011fe86c2619dba2cab747 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Thu, 13 Jul 2023 22:33:13 +0800 Subject: [PATCH 10/20] Update the exchangis_deploy_cn.md --- docs/zh_CN/ch1/exchangis_deploy_cn.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/zh_CN/ch1/exchangis_deploy_cn.md b/docs/zh_CN/ch1/exchangis_deploy_cn.md index 1ff34d0d2..6c7376e96 100644 --- a/docs/zh_CN/ch1/exchangis_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_deploy_cn.md @@ -6,6 +6,7 @@ Exchangis 的安装，主要分为以下四步： 2. Exchangis 安装部署 3. DSS ExchangisAppConn 安装部署 4. Linkis Sqoop 引擎安装部署 +5. Linkis DataX 引擎安装部署 ## 1. Exchangis 依赖环境准备 @@ -277,6 +278,10 @@ Exchangis 已默认提供了编译好的前端安装包，可直接下载使用 如您想正常执行 Exchangis1.0.0 的 Sqoop作业，还需安装 Linkis Sqoop 引擎，请参考: [Linkis Sqoop 引擎插件安装文档](https://linkis.staged.apache.org/zh-CN/docs/1.1.2/engine_usage/sqoop) -## 5. 如何登录使用 Exchangis +## 5. Linkis DataX 引擎安装部署 + +如您想正常执行 Exchangis1.0.0 的 DataX作业，还需安装 Linkis DataX 引擎，请参考: [Linkis DataX 引擎插件安装文档](https://linkis.staged.apache.org/zh-CN/docs/1.1.2/engine_usage/datax)（TODO：待Linkis确认） + +## 6. 如何登录使用 Exchangis Exchangis1.0更多使用说明，请参考用户使用手册[Exchangis1.0 用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_user_manual_cn.md) From e84847ed93a8d21aba9df7111cb21e1eb2b1f214 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Tue, 18 Jul 2023 16:00:51 +0800 Subject: [PATCH 11/20] Adjust the dependencies between modules;Upgrade the version of dss to 1.1.2;Upgrade the version of linkis to 1.4.0-SNAPSHOT --- exchangis-dao/pom.xml | 12 +- .../loader/ExchangisDataSourceLoader.java | 1 - .../service/LinkisMetadataInfoService.java | 4 +- .../LocalExchangisDataSourceLoader.java | 5 +- .../service/ExchangisDataSourceService.java | 4 +- .../engineconn-plugins/datax/pom.xml | 2 +- .../datax/DataxEngineConnPlugin.scala | 10 +- .../engineconn-plugins/sqoop/pom.xml | 7 +- .../sqoop/SqoopEngineConnPlugin.scala | 8 +- .../impl/ExchangisJobDsBindServiceImpl.java | 3 +- exchangis-job/exchangis-job-service/pom.xml | 2 +- .../exchangis-project-provider/pom.xml | 11 ++ .../ExchangisJobDssAppConnRestfulApi.java | 109 ---------- .../ExchangisProjectDssAppConnRestfulApi.java | 186 +++++++----------- ...eExchangisProjectDssAppConnRestfulApi.java | 161 +++++++++++++++ .../impl/ProjectExportServiceImpl.java | 2 + exchangis-server/pom.xml | 7 +- pom.xml | 35 ++-- 18 files changed, 293 insertions(+), 276 deletions(-) delete mode 100644 exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisJobDssAppConnRestfulApi.java create mode 100644 exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/PreExchangisProjectDssAppConnRestfulApi.java diff --git a/exchangis-dao/pom.xml b/exchangis-dao/pom.xml index c568837c6..5776a5b52 100644 --- a/exchangis-dao/pom.xml +++ b/exchangis-dao/pom.xml @@ -21,24 +21,20 @@ org.apache.linkis linkis-mybatis - ${linkis.version} org.apache.linkis linkis-module - ${linkis.version} - - - validation-api - javax.validation - - org.hibernate hibernate-validator ${hibernate.validator} + + org.springframework + spring-orm + diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java index 60400f70f..f68f696e8 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java @@ -10,7 +10,6 @@ public interface ExchangisDataSourceLoader { -// String EXCHANGIS_DIR_NAME = "exchangis-extds"; String EXCHANGIS_DIR_NAME = Objects.isNull(CommonVars.apply("wds.exchangis.datasource.extension.dir").getValue()) ? "exchangis-extds" : CommonVars.apply("wds.exchangis.datasource.extension.dir").getValue().toString(); String PROPERTIES_NAME = "extds.properties"; diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java index 4237037b5..5bbe000e2 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java +++ b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java @@ -3,11 +3,9 @@ import com.webank.wedatasphere.exchangis.common.EnvironmentUtils; import com.webank.wedatasphere.exchangis.datasource.core.domain.MetaColumn; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; -import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisServiceRpcException; import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; import com.webank.wedatasphere.exchangis.datasource.core.service.rpc.ServiceRpcClient; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisLinkisRemoteClient; -//import com.webank.wedatasphere.exchangis.datasource.linkis.partition.MetadataGetPartitionsResult; import com.webank.wedatasphere.exchangis.datasource.linkis.request.MetadataGetConnInfoAction; import com.webank.wedatasphere.exchangis.datasource.linkis.request.MetadataGetPartitionPropsAction; import com.webank.wedatasphere.exchangis.datasource.linkis.response.MetadataGetConnInfoResult; @@ -21,7 +19,7 @@ import org.apache.linkis.datasource.client.response.MetadataGetColumnsResult; import org.apache.linkis.datasource.client.response.MetadataGetPartitionsResult; import org.apache.linkis.datasource.client.response.MetadataGetTablePropsResult; -import org.apache.linkis.metadatamanager.common.domain.MetaColumnInfo; +import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import java.util.*; diff --git a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java index b29873bf9..50f6f59e3 100644 --- a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java +++ b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java @@ -7,6 +7,7 @@ import com.webank.wedatasphere.exchangis.datasource.loader.clazzloader.ExchangisDataSourceClassLoader; import com.webank.wedatasphere.exchangis.datasource.loader.utils.ExceptionHelper; import com.webank.wedatasphere.exchangis.datasource.loader.utils.ExtDsUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.linkis.common.exception.ErrorException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +39,9 @@ public void init(MapperHook mapperHook) throws Exception { // 初始化磁盘扫描加载 ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); String loadClassPath = Objects.requireNonNull(currentClassLoader.getResource(EXCHANGIS_DIR_NAME)).getPath(); -// String libPathUrl = loadClassPath + ".." + File.separator + ".." + File.separator + EXCHANGIS_DIR_NAME; + if (StringUtils.endsWith(loadClassPath, File.separator)) { + loadClassPath = loadClassPath + File.separator; + } String libPathUrl = loadClassPath + ".." + File.separator + EXCHANGIS_DIR_NAME; LOGGER.info("libPath url is {}", libPathUrl); List jars = ExtDsUtils.getJarsUrlsOfPath(libPathUrl); diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java index 7b313c381..40fcced7b 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java @@ -43,7 +43,7 @@ import org.apache.linkis.datasourcemanager.common.exception.JsonErrorException; import org.apache.linkis.datasourcemanager.common.util.json.Json; import org.apache.linkis.httpclient.response.Result; -import org.apache.linkis.metadatamanager.common.domain.MetaColumnInfo; +import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import org.apache.linkis.server.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -401,7 +401,7 @@ public Message deleteDataSource(HttpServletRequest request, /*String type,*/ Lon QueryWrapper condition = new QueryWrapper<>(); condition.eq("source_ds_id", id).or().eq("sink_ds_id", id); - Integer inUseCount = this.exchangisJobDsBindMapper.selectCount(condition); + Long inUseCount = this.exchangisJobDsBindMapper.selectCount(condition); if (inUseCount > 0) { throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_DATASOURCE_DELETE_ERROR.getCode(), "目前存在引用依赖"); } diff --git a/exchangis-engines/engineconn-plugins/datax/pom.xml b/exchangis-engines/engineconn-plugins/datax/pom.xml index 13c16c435..392671b64 100644 --- a/exchangis-engines/engineconn-plugins/datax/pom.xml +++ b/exchangis-engines/engineconn-plugins/datax/pom.xml @@ -13,7 +13,7 @@ linkis-engineplugin-datax - 1.1.6-webank + 1.4.0-SNAPSHOT 3.0.0-Plus-2 3.0.0 2.7.2 diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala index 34f6c4e4a..fb3ac3f68 100644 --- a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala +++ b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala @@ -26,7 +26,9 @@ import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuil import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceFactory import org.apache.linkis.manager.label.entity.Label -import java.util +import java.util.Map +import java.util.List +import java.util.ArrayList class DataxEngineConnPlugin extends EngineConnPlugin { @@ -36,7 +38,7 @@ class DataxEngineConnPlugin extends EngineConnPlugin { private var engineConnFactory: EngineConnFactory = _ private val engineConnFactoryLocker = new Array[Byte](0) - override def init(params: util.Map[String, Any]): Unit = {} + override def init(params: Map[String, AnyRef]): Unit = {} override def getEngineResourceFactory: EngineResourceFactory = { if (null == engineResourceFactory) engineResourceFactoryLocker.synchronized { @@ -60,7 +62,7 @@ class DataxEngineConnPlugin extends EngineConnPlugin { engineConnFactory } - override def getDefaultLabels: util.List[Label[_]] = { - new util.ArrayList[Label[_]]() + override def getDefaultLabels: List[Label[_]] = { + new ArrayList[Label[_]]() } } diff --git a/exchangis-engines/engineconn-plugins/sqoop/pom.xml b/exchangis-engines/engineconn-plugins/sqoop/pom.xml index 7c8f8cde5..8e8559c48 100644 --- a/exchangis-engines/engineconn-plugins/sqoop/pom.xml +++ b/exchangis-engines/engineconn-plugins/sqoop/pom.xml @@ -32,7 +32,7 @@ 1.4.6 3.1.2 2.7.2 - 1.1.6-webank + 1.4.0-SNAPSHOT @@ -187,28 +187,24 @@ linkis-engineconn-plugin-core ${linkis.version} - org.apache.linkis linkis-rpc ${linkis.version} provided - org.apache.linkis linkis-storage ${linkis.version} provided - org.apache.linkis linkis-common ${linkis.version} provided - @@ -217,7 +213,6 @@ org.apache.maven.plugins maven-deploy-plugin - net.alchim31.maven scala-maven-plugin diff --git a/exchangis-engines/engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/SqoopEngineConnPlugin.scala b/exchangis-engines/engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/SqoopEngineConnPlugin.scala index 52266bb71..ae322a3f5 100644 --- a/exchangis-engines/engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/SqoopEngineConnPlugin.scala +++ b/exchangis-engines/engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/SqoopEngineConnPlugin.scala @@ -25,16 +25,17 @@ import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.engineconnplugin.sqoop.factory.SqoopEngineConnFactory import org.apache.linkis.engineconnplugin.sqoop.launch.SqoopEngineConnLaunchBuilder +import java.util.Map +import java.util.List class SqoopEngineConnPlugin extends EngineConnPlugin{ private val EP_CONTEXT_CONSTRUCTOR_LOCK = new Object() private var engineResourceFactory: EngineResourceFactory = _ private var engineConnLaunchBuilder: EngineConnLaunchBuilder = _ private var engineConnFactory: EngineConnFactory = _ - override def init(params: java.util.Map[String, Any]): Unit = {} + override def init(params: Map[String, AnyRef]): Unit = {} override def getEngineResourceFactory: EngineResourceFactory = { - EP_CONTEXT_CONSTRUCTOR_LOCK.synchronized{ if(null == engineResourceFactory){ engineResourceFactory = new GenericEngineResourceFactory @@ -62,5 +63,6 @@ class SqoopEngineConnPlugin extends EngineConnPlugin{ } } - override def getDefaultLabels: java.util.List[Label[_]] = new java.util.ArrayList[Label[_]] + override def getDefaultLabels: List[Label[_]] = new java.util.ArrayList[Label[_]] + } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java index d55be86f8..567895e08 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java @@ -26,14 +26,13 @@ public void updateJobDsBind(Long jobId, List dsBinds) { for (ExchangisJobDsBind dsBind : dsBinds) { this.dsBindMapper.insert(dsBind); } - } @Override public boolean inUse(Long datasourceId) { QueryWrapper condition = new QueryWrapper<>(); condition.eq("source_ds_id", datasourceId).or().eq("sink_ds_id", datasourceId); - Integer count = Optional.ofNullable(this.dsBindMapper.selectCount(condition)).orElse(0); + Long count = Optional.ofNullable(this.dsBindMapper.selectCount(condition)).orElse(0L); return count > 0; } } diff --git a/exchangis-job/exchangis-job-service/pom.xml b/exchangis-job/exchangis-job-service/pom.xml index c63896b36..61a1b2133 100644 --- a/exchangis-job/exchangis-job-service/pom.xml +++ b/exchangis-job/exchangis-job-service/pom.xml @@ -5,7 +5,7 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 diff --git a/exchangis-project/exchangis-project-provider/pom.xml b/exchangis-project/exchangis-project-provider/pom.xml index 0a94a8840..b658e8b07 100644 --- a/exchangis-project/exchangis-project-provider/pom.xml +++ b/exchangis-project/exchangis-project-provider/pom.xml @@ -24,4 +24,15 @@ + + + + src/main/java + + **/*.xml + + + + + \ No newline at end of file diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisJobDssAppConnRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisJobDssAppConnRestfulApi.java deleted file mode 100644 index b2d41933d..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisJobDssAppConnRestfulApi.java +++ /dev/null @@ -1,109 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.restful.external; - -import com.webank.wedatasphere.exchangis.common.AuditLogUtils; -import com.webank.wedatasphere.exchangis.common.UserUtils; -import com.webank.wedatasphere.exchangis.common.enums.OperateTypeEnum; -import com.webank.wedatasphere.exchangis.common.enums.TargetTypeEnum; -import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.project.provider.mapper.ProjectMapper; -import com.webank.wedatasphere.exchangis.project.server.service.ProjectCopyService; -import com.webank.wedatasphere.exchangis.project.server.service.impl.ProjectExportServiceImpl; -import com.webank.wedatasphere.exchangis.project.server.service.impl.ProjectImportServerImpl; -import org.apache.linkis.server.Message; -import org.apache.linkis.server.security.SecurityFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; - -import javax.annotation.Resource; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.core.Context; -import java.rmi.ServerException; -import java.util.Map; - -/** - * Define to support the app conn, in order to distinguish from the inner api - */ -@RestController -@RequestMapping(value = "/dss/exchangis/main/appJob", produces = {"application/json;charset=utf-8"}) -public class ExchangisJobDssAppConnRestfulApi { - - private static final Logger LOG = LoggerFactory.getLogger(ExchangisJobDssAppConnRestfulApi.class); - - @Resource - private ProjectImportServerImpl projectImportServer; - - @Resource - private ProjectExportServiceImpl projectExportService; - - @Resource - private ProjectCopyService projectCopyService; - - @Autowired - private ProjectMapper projectMapper; - - @RequestMapping(value = "/import", method = RequestMethod.POST) - public Message importJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { - - Message response = null; - String userName = UserUtils.getLoginUser(request); - String oringinUser = SecurityFilter.getLoginUsername(request); - try { - LOG.info("param: {}", params); - /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { - return Message.error("You have no permission to import (没有导入权限)"); - }*/ - response = projectImportServer.importProject(request, params); - LOG.info("import job success"); - } catch (ExchangisJobServerException e) { - String message = "Fail import job [ id: " + params + "] (导入任务失败)"; - LOG.error(message, e); - response = Message.error(message); - } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.IMPORT, request); - return response; - - } - - @RequestMapping(value = "/export", method = RequestMethod.POST) - public Message exportJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { - String userName = UserUtils.getLoginUser(request); - - LOG.info("export function params: {}", params); - String oringinUser = SecurityFilter.getLoginUsername(request); - Message response = null; - try { - /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { - return Message.error("You have no permission to export (没有导出权限)"); - }*/ - response = projectExportService.exportProject(params, userName, request); - LOG.info("export job success"); - } catch (Exception e) { - String message = "Fail Export job [ id: " + params + "] (导出任务失败)"; - LOG.error(message, e); - response = Message.error(message); - } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.EXPORT, request); - return response; - } - - @RequestMapping(value = "/copy", method = RequestMethod.POST) - public Message copy(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException { - String userName = UserUtils.getLoginUser(request); - String oringinUser = SecurityFilter.getLoginUsername(request); - - LOG.info("copy function params: {}", params); - Message response = null; - try { - response = projectCopyService.copy(params, userName, request); - LOG.info("copy node success"); - } catch (Exception e) { - String message = "Fail Copy project [ id: " + params + "] (导出任务失败)"; - LOG.error(message, e); - response = Message.error(message); - } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Copy parameter is: " + params.toString(), OperateTypeEnum.COPY, request); - return response; - } -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java index 017d6571a..6221c71e1 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java @@ -4,158 +4,106 @@ import com.webank.wedatasphere.exchangis.common.UserUtils; import com.webank.wedatasphere.exchangis.common.enums.OperateTypeEnum; import com.webank.wedatasphere.exchangis.common.enums.TargetTypeEnum; -import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; -import com.webank.wedatasphere.exchangis.project.server.domain.OperationType; -import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; -import com.webank.wedatasphere.exchangis.project.server.utils.ExchangisProjectRestfulUtils; -import com.webank.wedatasphere.exchangis.project.server.utils.ProjectAuthorityUtils; -import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectAppVo; -import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.math3.util.Pair; -import org.apache.linkis.common.utils.JsonUtils; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.project.provider.mapper.ProjectMapper; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectCopyService; +import com.webank.wedatasphere.exchangis.project.server.service.impl.ProjectExportServiceImpl; +import com.webank.wedatasphere.exchangis.project.server.service.impl.ProjectImportServerImpl; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.validation.BindingResult; -import org.springframework.validation.annotation.Validated; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; -import javax.validation.groups.Default; +import javax.ws.rs.core.Context; +import java.rmi.ServerException; +import java.util.Map; /** - * Restful class for dss project + * Define to support the app conn, in order to distinguish from the inner api */ -@Deprecated @RestController -@RequestMapping(value = "/dss/exchangis/main/appProject", produces = {"application/json;charset=utf-8"}) +@RequestMapping(value = "/dss/exchangis/main/appJob", produces = {"application/json;charset=utf-8"}) public class ExchangisProjectDssAppConnRestfulApi { + private static final Logger LOG = LoggerFactory.getLogger(ExchangisProjectDssAppConnRestfulApi.class); - /** - * Project service - */ @Resource - private ProjectService projectService; - - @RequestMapping(value = "", method = RequestMethod.POST) - public Message createProject(@Validated @RequestBody ExchangisProjectAppVo project, - BindingResult result, HttpServletRequest request){ - ExchangisProjectInfo projectVo = new ExchangisProjectInfo(project); - if (result.hasErrors()){ - return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); - } + private ProjectImportServerImpl projectImportServer; - String oringinUser = SecurityFilter.getLoginUsername(request); - String username = UserUtils.getLoginUser(request); - if (StringUtils.isBlank(projectVo.getViewUsers()) || !StringUtils.contains(projectVo.getViewUsers(), username)) { - projectVo.setViewUsers(username + projectVo.getViewUsers()); - } - if (StringUtils.isBlank(projectVo.getEditUsers()) || !StringUtils.contains(projectVo.getEditUsers(), username)) { - projectVo.setEditUsers(username + projectVo.getEditUsers()); - } - if (StringUtils.isBlank(projectVo.getExecUsers()) || !StringUtils.contains(projectVo.getExecUsers(), username)) { - projectVo.setExecUsers(username + projectVo.getExecUsers()); + @Resource + private ProjectExportServiceImpl projectExportService; - } + @Resource + private ProjectCopyService projectCopyService; - try { - LOG.info("CreateProject from DSS AppConn, vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); - if (projectService.existsProject(null, projectVo.getName())){ - return Message.error("Have the same name project (存在同名工程)"); - } - long projectIdd = projectService.createProject(projectVo, username); - String projectId = String.valueOf(projectIdd); - AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, String.valueOf(projectId), "Project name is: " + projectVo.getName(), OperateTypeEnum.CREATE, request); - return ExchangisProjectRestfulUtils.dealOk("创建工程成功", - new Pair<>("projectName", projectVo.getName()), - new Pair<>("projectId", projectId)); - } catch (Exception t) { - LOG.error("Failed to create project for user {} from DSS", username, t); - return Message.error("Fail to create project from DSS(创建工程失败)"); - } - } + @Autowired + private ProjectMapper projectMapper; + @RequestMapping(value = "/import", method = RequestMethod.POST) + public Message importJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { - /** - * Update project - * @param request request - * @param projectVo project vo - * @return - */ - @RequestMapping( value = "/{id:\\d+}", method = RequestMethod.PUT) - public Message updateProject(@PathVariable("id") Long id, @Validated({UpdateGroup.class, Default.class}) @RequestBody ExchangisProjectInfo projectVo - , BindingResult result, HttpServletRequest request) { - if (result.hasErrors()){ - return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); - } - String username = UserUtils.getLoginUser(request); + Message response = null; + String userName = UserUtils.getLoginUser(request); String oringinUser = SecurityFilter.getLoginUsername(request); try { - ExchangisProjectInfo projectStored = projectService.getProjectDetailById(Long.valueOf(projectVo.getId())); - if (!ProjectAuthorityUtils.hasProjectAuthority(username, projectStored, OperationType.PROJECT_ALTER)) { - return Message.error("You have no permission to update (没有项目的更新权限)"); - } - - LOG.info("UpdateProject vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); - projectService.updateProject(projectVo, username); - AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, id.toString(), "Project name is: " + projectVo.getName(), OperateTypeEnum.UPDATE, request); - return ExchangisProjectRestfulUtils.dealOk("更新工程成功", - new Pair<>("projectName", projectVo.getName()), - new Pair<>("projectId", projectVo.getId())); - } catch (Exception t) { - LOG.error("Failed to update project for user {}", username, t); - return Message.error("Fail to update project (更新工程失败)"); + LOG.info("param: {}", params); + /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { + return Message.error("You have no permission to import (没有导入权限)"); + }*/ + response = projectImportServer.importProject(request, params); + LOG.info("import job success"); + } catch (ExchangisJobServerException e) { + String message = "Fail import job [ id: " + params + "] (导入任务失败)"; + LOG.error(message, e); + response = Message.error(message); } + AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.IMPORT, request); + return response; + } - /** - * Delete project - * @param request http request - * @param name project name - * @return - */ - @RequestMapping( value = "/{name}", method = RequestMethod.POST) - public Message deleteProject(HttpServletRequest request, @PathVariable("name") String name) { - String username = UserUtils.getLoginUser(request); + @RequestMapping(value = "/export", method = RequestMethod.POST) + public Message exportJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { + String userName = UserUtils.getLoginUser(request); + + LOG.info("export function params: {}", params); String oringinUser = SecurityFilter.getLoginUsername(request); + Message response = null; try { - ExchangisProjectInfo projectInfo = projectService.selectByName(name); - if (!ProjectAuthorityUtils.hasProjectAuthority(username, projectInfo, OperationType.PROJECT_ALTER)) { - return Message.error("You have no permission to delete (删除项目失败)"); - } - - projectService.deleteProjectByName(name); - AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, "", "Project name is: " + name, OperateTypeEnum.DELETE, request); - return ExchangisProjectRestfulUtils.dealOk("删除工程成功"); - } catch (Exception t) { - LOG.error("Failed to delete project for user {}", username, t); - return Message.error("Failed to delete project (删除工程失败)"); + /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { + return Message.error("You have no permission to export (没有导出权限)"); + }*/ + response = projectExportService.exportProject(params, userName, request); + LOG.info("export job success"); + } catch (Exception e) { + String message = "Fail Export job [ id: " + params + "] (导出任务失败)"; + LOG.error(message, e); + response = Message.error(message); } - + AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.EXPORT, request); + return response; } + @RequestMapping(value = "/copy", method = RequestMethod.POST) + public Message copy(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException { + String userName = UserUtils.getLoginUser(request); + String oringinUser = SecurityFilter.getLoginUsername(request); - /** - * check project name - * @param request http request - * @param name project name - * @return - */ - @RequestMapping( value = "/check/{name}", method = RequestMethod.POST) - public Message getProjectByName(HttpServletRequest request, @PathVariable("name") String name) { - String username = UserUtils.getLoginUser(request); + LOG.info("copy function params: {}", params); + Message response = null; try { - ExchangisProjectInfo projectInfo = projectService.selectByName(name); - return ExchangisProjectRestfulUtils.dealOk("根据名字获取工程成功", - new Pair<>("projectInfo",projectInfo)); - } catch (Exception t) { - LOG.error("Failed to delete project for user {}", username, t); - return Message.error("Failed to delete project (根据名字获取工程失败)"); + response = projectCopyService.copy(params, userName, request); + LOG.info("copy node success"); + } catch (Exception e) { + String message = "Fail Copy project [ id: " + params + "] (导出任务失败)"; + LOG.error(message, e); + response = Message.error(message); } + AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Copy parameter is: " + params.toString(), OperateTypeEnum.COPY, request); + return response; } - } diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/PreExchangisProjectDssAppConnRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/PreExchangisProjectDssAppConnRestfulApi.java new file mode 100644 index 000000000..0212d6b3d --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/PreExchangisProjectDssAppConnRestfulApi.java @@ -0,0 +1,161 @@ +//package com.webank.wedatasphere.exchangis.project.server.restful.external; +// +//import com.webank.wedatasphere.exchangis.common.AuditLogUtils; +//import com.webank.wedatasphere.exchangis.common.UserUtils; +//import com.webank.wedatasphere.exchangis.common.enums.OperateTypeEnum; +//import com.webank.wedatasphere.exchangis.common.enums.TargetTypeEnum; +//import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; +//import com.webank.wedatasphere.exchangis.project.server.domain.OperationType; +//import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; +//import com.webank.wedatasphere.exchangis.project.server.utils.ExchangisProjectRestfulUtils; +//import com.webank.wedatasphere.exchangis.project.server.utils.ProjectAuthorityUtils; +//import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectAppVo; +//import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; +//import org.apache.commons.lang3.StringUtils; +//import org.apache.commons.math3.util.Pair; +//import org.apache.linkis.common.utils.JsonUtils; +//import org.apache.linkis.server.Message; +//import org.apache.linkis.server.security.SecurityFilter; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +//import org.springframework.validation.BindingResult; +//import org.springframework.validation.annotation.Validated; +//import org.springframework.web.bind.annotation.*; +// +//import javax.annotation.Resource; +//import javax.servlet.http.HttpServletRequest; +//import javax.validation.groups.Default; +// +///** +// * Restful class for dss project +// */ +//@Deprecated +//@RestController +//@RequestMapping(value = "/dss/exchangis/main/appProject", produces = {"application/json;charset=utf-8"}) +//public class PreExchangisProjectDssAppConnRestfulApi { +// private static final Logger LOG = LoggerFactory.getLogger(PreExchangisProjectDssAppConnRestfulApi.class); +// +// /** +// * Project service +// */ +// @Resource +// private ProjectService projectService; +// +// @RequestMapping(value = "", method = RequestMethod.POST) +// public Message createProject(@Validated @RequestBody ExchangisProjectAppVo project, +// BindingResult result, HttpServletRequest request){ +// ExchangisProjectInfo projectVo = new ExchangisProjectInfo(project); +// if (result.hasErrors()){ +// return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); +// } +// +// String oringinUser = SecurityFilter.getLoginUsername(request); +// String username = UserUtils.getLoginUser(request); +// if (StringUtils.isBlank(projectVo.getViewUsers()) || !StringUtils.contains(projectVo.getViewUsers(), username)) { +// projectVo.setViewUsers(username + projectVo.getViewUsers()); +// } +// if (StringUtils.isBlank(projectVo.getEditUsers()) || !StringUtils.contains(projectVo.getEditUsers(), username)) { +// projectVo.setEditUsers(username + projectVo.getEditUsers()); +// } +// if (StringUtils.isBlank(projectVo.getExecUsers()) || !StringUtils.contains(projectVo.getExecUsers(), username)) { +// projectVo.setExecUsers(username + projectVo.getExecUsers()); +// +// } +// +// try { +// LOG.info("CreateProject from DSS AppConn, vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); +// if (projectService.existsProject(null, projectVo.getName())){ +// return Message.error("Have the same name project (存在同名工程)"); +// } +// long projectIdd = projectService.createProject(projectVo, username); +// String projectId = String.valueOf(projectIdd); +// AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, String.valueOf(projectId), "Project name is: " + projectVo.getName(), OperateTypeEnum.CREATE, request); +// return ExchangisProjectRestfulUtils.dealOk("创建工程成功", +// new Pair<>("projectName", projectVo.getName()), +// new Pair<>("projectId", projectId)); +// } catch (Exception t) { +// LOG.error("Failed to create project for user {} from DSS", username, t); +// return Message.error("Fail to create project from DSS(创建工程失败)"); +// } +// } +// +// +// /** +// * Update project +// * @param request request +// * @param projectVo project vo +// * @return +// */ +// @RequestMapping( value = "/{id:\\d+}", method = RequestMethod.PUT) +// public Message updateProject(@PathVariable("id") Long id, @Validated({UpdateGroup.class, Default.class}) @RequestBody ExchangisProjectInfo projectVo +// , BindingResult result, HttpServletRequest request) { +// if (result.hasErrors()){ +// return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); +// } +// String username = UserUtils.getLoginUser(request); +// String oringinUser = SecurityFilter.getLoginUsername(request); +// try { +// ExchangisProjectInfo projectStored = projectService.getProjectDetailById(Long.valueOf(projectVo.getId())); +// if (!ProjectAuthorityUtils.hasProjectAuthority(username, projectStored, OperationType.PROJECT_ALTER)) { +// return Message.error("You have no permission to update (没有项目的更新权限)"); +// } +// +// LOG.info("UpdateProject vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); +// projectService.updateProject(projectVo, username); +// AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, id.toString(), "Project name is: " + projectVo.getName(), OperateTypeEnum.UPDATE, request); +// return ExchangisProjectRestfulUtils.dealOk("更新工程成功", +// new Pair<>("projectName", projectVo.getName()), +// new Pair<>("projectId", projectVo.getId())); +// } catch (Exception t) { +// LOG.error("Failed to update project for user {}", username, t); +// return Message.error("Fail to update project (更新工程失败)"); +// } +// } +// +// /** +// * Delete project +// * @param request http request +// * @param name project name +// * @return +// */ +// @RequestMapping( value = "/{name}", method = RequestMethod.POST) +// public Message deleteProject(HttpServletRequest request, @PathVariable("name") String name) { +// String username = UserUtils.getLoginUser(request); +// String oringinUser = SecurityFilter.getLoginUsername(request); +// try { +// ExchangisProjectInfo projectInfo = projectService.selectByName(name); +// if (!ProjectAuthorityUtils.hasProjectAuthority(username, projectInfo, OperationType.PROJECT_ALTER)) { +// return Message.error("You have no permission to delete (删除项目失败)"); +// } +// +// projectService.deleteProjectByName(name); +// AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, "", "Project name is: " + name, OperateTypeEnum.DELETE, request); +// return ExchangisProjectRestfulUtils.dealOk("删除工程成功"); +// } catch (Exception t) { +// LOG.error("Failed to delete project for user {}", username, t); +// return Message.error("Failed to delete project (删除工程失败)"); +// } +// +// } +// +// +// /** +// * check project name +// * @param request http request +// * @param name project name +// * @return +// */ +// @RequestMapping( value = "/check/{name}", method = RequestMethod.POST) +// public Message getProjectByName(HttpServletRequest request, @PathVariable("name") String name) { +// String username = UserUtils.getLoginUser(request); +// try { +// ExchangisProjectInfo projectInfo = projectService.selectByName(name); +// return ExchangisProjectRestfulUtils.dealOk("根据名字获取工程成功", +// new Pair<>("projectInfo",projectInfo)); +// } catch (Exception t) { +// LOG.error("Failed to delete project for user {}", username, t); +// return Message.error("Failed to delete project (根据名字获取工程失败)"); +// } +// } +// +//} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectExportServiceImpl.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectExportServiceImpl.java index 3fcd0f1d6..2abc89864 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectExportServiceImpl.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectExportServiceImpl.java @@ -19,6 +19,7 @@ import org.apache.linkis.server.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; @@ -32,6 +33,7 @@ * @author jefftlin * @date 2023/7/13 */ +@Service public class ProjectExportServiceImpl implements ProjectExportService { private static final Logger LOG = LoggerFactory.getLogger(ProjectExportServiceImpl.class); diff --git a/exchangis-server/pom.xml b/exchangis-server/pom.xml index e805d9559..2b16d972d 100644 --- a/exchangis-server/pom.xml +++ b/exchangis-server/pom.xml @@ -28,8 +28,12 @@ ${exchangis.version} - linkis-module org.apache.linkis + linkis-module + + + spring-jdbc + org.springframework @@ -60,7 +64,6 @@ com.fasterxml classmate 1.5.1 - diff --git a/pom.xml b/pom.xml index 842bc08ca..587123131 100644 --- a/pom.xml +++ b/pom.xml @@ -38,11 +38,12 @@ 1.1.2 - 1.1.0 - 1.1.6-webank - 1.1.1 + 1.1.2 + 1.4.0-SNAPSHOT + 1.4.0-SNAPSHOT 0.1.0-SNAPSHOT - 2.11.8 + 2.12.17 + 4.7.1 1.8 3.3.3 2.8.5 @@ -67,7 +68,6 @@ 0.1.0-SNAPSHOT - exchangis-dao exchangis-project @@ -109,10 +109,17 @@ org.apache.linkis linkis-mybatis ${linkis.version} + + + org.springframework + spring-orm + + org.apache.linkis linkis-module + ${linkis.version} org.springframework.boot @@ -123,20 +130,22 @@ org.hibernate.validator - ${linkis.version} org.apache.linkis linkis-common ${linkis.version} - org.apache.linkis linkis-protocol ${linkis.version} - + + org.apache.linkis + linkis-datasource-client + ${linkis.datasource.version} + com.google.code.gson gson @@ -162,11 +171,10 @@ com.thoughtworks.xstream ${xstream.version} - - org.apache.linkis - linkis-datasource-client - ${linkis.datasource.version} + org.springframework + spring-orm + 5.2.15.RELEASE @@ -225,7 +233,7 @@ net.alchim31.maven scala-maven-plugin - 3.2.2 + ${scala-maven-plugin.version} eclipse-add-source @@ -258,7 +266,6 @@ ${scala.version} incremental - true From 4c202300c37bba2c3a1f5333f86b10c86cc4ab38 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Tue, 18 Jul 2023 16:04:52 +0800 Subject: [PATCH 12/20] Upgrade installation scripts and documents --- .../config/exchangis-server.properties | 8 +- assembly-package/sbin/common.sh | 2 +- assembly-package/sbin/daemon.sh | 2 +- assembly-package/sbin/env.properties | 8 +- assembly-package/sbin/install.sh | 63 ++--- assembly-package/sbin/launcher.sh | 11 +- db/1.1.1/exchangis_ddl.sql | 253 ++++++++++++++++++ db/1.1.1/exchangis_dml.sql | 81 ++++++ db/1.1.2/exchangis_ddl.sql | 1 + db/exchangis_ddl.sql | 4 +- docs/en_US/ch1/component_upgrade_en.md | 6 +- docs/en_US/ch1/exchangis_datasource_en.md | 2 +- docs/en_US/ch1/exchangis_deploy_en.md | 14 +- docs/en_US/ch1/exchangis_sqoop_deploy_en.md | 2 +- docs/zh_CN/ch1/component_upgrade_cn.md | 12 +- docs/zh_CN/ch1/exchangis_deploy_cn.md | 65 +++-- .../src/main/resources/datax.icon | 1 + .../src/main/resources/init.sql | 4 +- .../src/main/resources/exchangis.properties | 2 +- 19 files changed, 439 insertions(+), 102 deletions(-) create mode 100644 db/1.1.1/exchangis_ddl.sql create mode 100644 db/1.1.1/exchangis_dml.sql create mode 100644 db/1.1.2/exchangis_ddl.sql create mode 100644 exchangis-plugins/exchangis-appconn/src/main/resources/datax.icon diff --git a/assembly-package/config/exchangis-server.properties b/assembly-package/config/exchangis-server.properties index f2c12f8bc..75bf44476 100644 --- a/assembly-package/config/exchangis-server.properties +++ b/assembly-package/config/exchangis-server.properties @@ -16,9 +16,7 @@ # wds.linkis.server.mybatis.datasource.url=jdbc:mysql://{IP}:{PORT}/{database}?useSSL=false&characterEncoding=UTF-8&allowMultiQueries=true&useAffectedRows=true - wds.linkis.server.mybatis.datasource.username={username} - wds.linkis.server.mybatis.datasource.password={password} wds.linkis.log.clear=true @@ -40,13 +38,17 @@ wds.exchangis.datasource.extension.dir=exchangis-extds/ wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.exchangis.datasource.server.restful.api,\ com.webank.wedatasphere.exchangis.project.server.restful,\ com.webank.wedatasphere.exchangis.job.server.restful -wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/workflow/dao/impl/*.xml,\ + +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,\ +classpath*:com/webank/wedatasphere/dss/workflow/dao/impl/*.xml,\ classpath*:com/webank/wedatasphere/exchangis/job/server/mapper/impl/*.xml,\ classpath*:com/webank/wedatasphere/exchangis/project/server/mapper/impl/*.xml,\ +classpath*:com/webank/wedatasphere/exchangis/project/provider/mapper/impl/*.xml,\ classpath*:com/webank/wedatasphere/exchangis/engine/server/mapper/*.xml wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.exchangis.dao,\ com.webank.wedatasphere.exchangis.project.server.mapper,\ + com.webank.wedatasphere.exchangis.project.provider.mapper,\ com.webank.wedatasphere.linkis.configuration.dao,\ com.webank.wedatasphere.dss.framework.appconn.dao,\ com.webank.wedatasphere.dss.workflow.dao,\ diff --git a/assembly-package/sbin/common.sh b/assembly-package/sbin/common.sh index 03d4e4666..8ee615b64 100644 --- a/assembly-package/sbin/common.sh +++ b/assembly-package/sbin/common.sh @@ -16,4 +16,4 @@ # declare -A MODULE_MAIN_CLASS -MODULE_MAIN_CLASS["exchangis-server"]="com.webank.wedatasphere.exchangis.server.boot.ExchangisServerApplication" +MODULE_MAIN_CLASS["dss-exchangis-main-server-dev"]="com.webank.wedatasphere.exchangis.server.boot.ExchangisServerApplication" diff --git a/assembly-package/sbin/daemon.sh b/assembly-package/sbin/daemon.sh index d86df111a..a21ccfab6 100644 --- a/assembly-package/sbin/daemon.sh +++ b/assembly-package/sbin/daemon.sh @@ -24,7 +24,7 @@ else fi usage(){ - echo "Usage is [start|stop|restart {service}]" + echo "Usage is [start|stop|restart {server}]" } start(){ diff --git a/assembly-package/sbin/env.properties b/assembly-package/sbin/env.properties index c849c950a..f849b4fa9 100644 --- a/assembly-package/sbin/env.properties +++ b/assembly-package/sbin/env.properties @@ -1,4 +1,4 @@ -EXCHANGIS_CONF_PATH="/appcom/config/exchangis-config" -EXCHANGIS_LOG_PATH="/appcom/logs/exchangis-log" -MODULE_DEFAULT_PREFIX="exchangis-server" -MODULE_DEFAULT_SUFFIX="" +EXCHANGIS_CONF_PATH=/appcom/config/exchangis-config/background +EXCHANGIS_LOG_PATH=/appcom/logs/exchangis/background +MODULE_DEFAULT_PREFIX="dss-exchangis-main-" +MODULE_DEFAULT_SUFFIX="-dev" diff --git a/assembly-package/sbin/install.sh b/assembly-package/sbin/install.sh index 16f453870..2d362038c 100644 --- a/assembly-package/sbin/install.sh +++ b/assembly-package/sbin/install.sh @@ -123,57 +123,36 @@ interact_echo(){ done } +# Initalize database init_database(){ -BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties" -# Start to initalize database -if [ "x${SQL_SOURCE_PATH}" != "x" ] && [ -f "${SQL_SOURCE_PATH}" ]; then - `mysql --version >/dev/null 2>&1` - interact_echo "Do you want to initalize database with sql?" - if [ $? == 0 ]; then - LOG INFO "\033[1m Scan out mysql command, so begin to initalize the database\033[0m" - #interact_echo "Do you want to initalize database with sql: [${SQL_SOURCE_PATH}]?" - #if [ $? == 0 ]; then + BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties" + if [ "x${SQL_SOURCE_PATH}" != "x" ] && [ -f "${SQL_SOURCE_PATH}" ]; then + `mysql --version >/dev/null 2>&1` DATASOURCE_URL="jdbc:mysql:\/\/${MYSQL_HOST}:${MYSQL_PORT}\/${DATABASE}\?useSSL=false\&characterEncoding=UTF-8\&allowMultiQueries=true" - mysql -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} --default-character-set=utf8 -e \ - "CREATE DATABASE IF NOT EXISTS ${DATABASE}; USE ${DATABASE}; source ${SQL_SOURCE_PATH};" - #sed -ri "s![#]?(DB_HOST=)\S*!\1${HOST}!g" ${BOOTSTRAP_PROP_FILE} - #sed -ri "s![#]?(DB_PORT=)\S*!\1${PORT}!g" ${BOOTSTRAP_PROP_FILE} sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.username=)\S*!\1${MYSQL_USERNAME}!g" ${BOOTSTRAP_PROP_FILE} sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.password=)\S*!\1${MYSQL_PASSWORD}!g" ${BOOTSTRAP_PROP_FILE} sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.url=)\S*!\1${DATASOURCE_URL}!g" ${BOOTSTRAP_PROP_FILE} - #fi - fi -fi + interact_echo "Do you want to initalize database with sql: [${SQL_SOURCE_PATH}]?" + if [ $? == 0 ]; then + LOG INFO "\033[1m Scan out mysql command, so begin to initalize the database\033[0m" + mysql -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} --default-character-set=utf8 -e \ + "CREATE DATABASE IF NOT EXISTS ${DATABASE}; USE ${DATABASE}; source ${SQL_SOURCE_PATH};" + fi + fi } init_properties(){ -BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties" -APPLICATION_YML="${CONF_PATH}/application-exchangis.yml" -# Start to initalize propertis - #interact_echo "Do you want to initalize exchangis-server.properties?" - #if [ $? == 0 ]; then - - LINKIS_GATEWAY_URL="http:\/\/${LINKIS_GATEWAY_HOST}:${LINKIS_GATEWAY_PORT}\/" - - if [ "x${LINKIS_SERVER_URL}" == "x" ]; then - LINKIS_SERVER_URL="http://127.0.0.1:3306" - fi - if [ "x${LINKIS_SERVER_URL}" == "x" ]; then - LINKIS_SERVER_URL="http://127.0.0.1:3306" - fi + BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties" + APPLICATION_YML="${CONF_PATH}/application-exchangis.yml" + LINKIS_GATEWAY_URL="http:\/\/${LINKIS_GATEWAY_HOST}:${LINKIS_GATEWAY_PORT}\/" + if [ "x${LINKIS_SERVER_URL}" == "x" ]; then + LINKIS_SERVER_URL="http://127.0.0.1:9001" + fi - sed -ri "s![#]?(wds.linkis.gateway.ip=)\S*!\1${LINKIS_GATEWAY_HOST}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(wds.linkis.gateway.port=)\S*!\1${LINKIS_GATEWAY_PORT}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(wds.linkis.gateway.url=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(wds.exchangis.datasource.client.serverurl=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(wds.exchangis.client.linkis.server-url=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE} - #sed -ri "s![#]?(wds.exchangis.datasource.client.authtoken.key=)\S*!\1${LINKIS_TOKEN}!g" ${BOOTSTRAP_PROP_FILE} - #sed -ri "s![#]?(wds.exchangis.datasource.client.authtoken.value=)\S*!\1${LINKIS_TOKEN}!g" ${BOOTSTRAP_PROP_FILE} - #sed -ri "s![#]?(wds.exchangis.client.linkis.token.value=)\S*!\1${LINKIS_TOKEN}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(wds.linkis.gateway.port=)\S*!\1${LINKIS_GATEWAY_PORT}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(port: )\S*!\1${EXCHANGIS_PORT}!g" ${APPLICATION_YML} - sed -ri "s![#]?(defaultZone: )\S*!\1${EUREKA_URL}!g" ${APPLICATION_YML} - #fi + sed -ri "s![#]?(wds.exchangis.datasource.client.serverurl=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE} + sed -ri "s![#]?(wds.exchangis.client.linkis.server-url=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE} + sed -ri "s![#]?(port: )\S*!\1${EXCHANGIS_PORT}!g" ${APPLICATION_YML} + sed -ri "s![#]?(defaultZone: )\S*!\1${EUREKA_URL}!g" ${APPLICATION_YML} } install_modules(){ diff --git a/assembly-package/sbin/launcher.sh b/assembly-package/sbin/launcher.sh index db6031428..50a79d279 100644 --- a/assembly-package/sbin/launcher.sh +++ b/assembly-package/sbin/launcher.sh @@ -86,8 +86,6 @@ load_env_definitions(){ fi } - - construct_java_command(){ verify_java_env if [[ "x${EXCHANGIS_CONF_PATH}" == "x" ]]; then @@ -114,7 +112,7 @@ construct_java_command(){ mkdir -p ${EXCHANGIS_PID_PATH} local classpath=${EXCHANGIS_CONF_PATH}":." local opts="" - classpath=${EXCHANGIS_LIB_PATH}/exchangis-server/*":"${classpath} + classpath=${EXCHANGIS_LIB_PATH}/"exchangis-server/*:"${classpath} LOG INFO "classpath:"${classpath} if [[ "x${EXCHANGIS_JAVA_OPTS}" == "x" ]]; then # Use G1 garbage collector @@ -175,7 +173,7 @@ wait_for_startup(){ return 0 fi sleep ${SLEEP_TIMEREVAL_S} - now_s=`date '+%s'` #计算当前时间时间戳 + now_s=`date '+%s'` done return 1 } @@ -204,14 +202,17 @@ launcher_start(){ fi construct_java_command $1 $2 # Execute + echo ${EXEC_JAVA} LOG INFO ${EXEC_JAVA} nohup ${EXEC_JAVA} >/dev/null 2>&1 & LOG INFO "Launcher: waiting [ $1 ] to start complete ..." wait_for_startup 20 $1 $2 if [[ $? -eq 0 ]]; then LOG INFO "Launcher: [ $1 ] start success" + LOG INFO ${EXCHANGIS_CONF_PATH} APPLICATION_YML="${EXCHANGIS_CONF_PATH}/application-exchangis.yml" EUREKA_URL=`cat ${APPLICATION_YML} | grep Zone | sed -n '1p'` + echo "${EUREKA_URL}" LOG INFO "Please check exchangis server in EUREKA_ADDRESS: ${EUREKA_URL#*:} " else LOG ERROR "Launcher: [ $1 ] start fail over 20 seconds, please retry it" @@ -249,4 +250,4 @@ launcher_stop(){ LOG ERROR "Launcher: [ $1 ] stop exceeded over 20s " >&2 return 1 fi -} \ No newline at end of file +} diff --git a/db/1.1.1/exchangis_ddl.sql b/db/1.1.1/exchangis_ddl.sql new file mode 100644 index 000000000..766a4eb37 --- /dev/null +++ b/db/1.1.1/exchangis_ddl.sql @@ -0,0 +1,253 @@ +-- exchangis_job_ds_bind definition +DROP TABLE IF EXISTS `exchangis_job_ds_bind`; +CREATE TABLE `exchangis_job_ds_bind` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_id` bigint(20) NOT NULL, + `task_index` int(11) NOT NULL, + `source_ds_id` bigint(20) NOT NULL, + `sink_ds_id` bigint(20) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=59575 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + +-- exchangis_job_entity definition +DROP TABLE IF EXISTS `exchangis_job_entity`; +CREATE TABLE `exchangis_job_entity` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(100) NOT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime(3) DEFAULT NULL, + `engine_type` varchar(45) DEFAULT '', + `job_labels` varchar(255) DEFAULT NULL, + `create_user` varchar(100) DEFAULT NULL, + `job_content` mediumtext, + `execute_user` varchar(100) DEFAULT '', + `job_params` text, + `job_desc` varchar(255) DEFAULT NULL, + `job_type` varchar(50) DEFAULT NULL, + `project_id` bigint(13) DEFAULT NULL, + `source` text, + `modify_user` varchar(50) DEFAULT NULL COMMENT '修改用户', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=5793 DEFAULT CHARSET=utf8; + + +-- exchangis_job_param_config definition +DROP TABLE IF EXISTS `exchangis_job_param_config`; +CREATE TABLE `exchangis_job_param_config` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `config_key` varchar(64) NOT NULL, + `config_name` varchar(64) NOT NULL, + `config_direction` varchar(16) DEFAULT NULL, + `type` varchar(32) NOT NULL, + `ui_type` varchar(32) DEFAULT NULL, + `ui_field` varchar(64) DEFAULT NULL, + `ui_label` varchar(32) DEFAULT NULL, + `unit` varchar(32) DEFAULT NULL, + `required` bit(1) DEFAULT b'0', + `value_type` varchar(32) DEFAULT NULL, + `value_range` varchar(255) DEFAULT NULL, + `default_value` varchar(255) DEFAULT NULL, + `validate_type` varchar(64) DEFAULT NULL, + `validate_range` varchar(64) DEFAULT NULL, + `validate_msg` varchar(255) DEFAULT NULL, + `is_hidden` bit(1) DEFAULT NULL, + `is_advanced` bit(1) DEFAULT NULL, + `source` varchar(255) DEFAULT NULL, + `level` tinyint(4) DEFAULT NULL, + `treename` varchar(32) DEFAULT NULL, + `sort` int(11) DEFAULT NULL, + `description` varchar(255) DEFAULT NULL, + `status` tinyint(4) DEFAULT NULL, + `ref_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=32 DEFAULT CHARSET=utf8; + +-- exchangis_project_info definition +DROP TABLE IF EXISTS `exchangis_project_info`; +-- udes_gzpc_pub_sit_01.exchangis_project_info definition +CREATE TABLE `exchangis_project_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(64) NOT NULL, + `description` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT DEFAULT CURRENT_TIMESTAMP, + `last_update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `create_user` varchar(64) DEFAULT NULL, + `last_update_user` varchar(64) DEFAULT NULL, + `project_labels` varchar(255) DEFAULT NULL, + `domain` varchar(32) DEFAULT NULL, + `exec_users` varchar(255) DEFAULT '', + `view_users` varchar(255) DEFAULT '', + `edit_users` varchar(255) DEFAULT '', + `source` text, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1497870871035974171 DEFAULT CHARSET=utf8; + +-- exchangis_project_user definition +DROP TABLE IF EXISTS `exchangis_project_user`; +CREATE TABLE `exchangis_project_user` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `project_id` bigint(20) NOT NULL, + `priv_user` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `priv` int(20) DEFAULT NULL, + `last_update_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `exchangis_project_user_un` (`project_id`) +) ENGINE=InnoDB AUTO_INCREMENT=844 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT; + +-- exchangis_launchable_task definition +DROP TABLE IF EXISTS `exchangis_launchable_task`; +CREATE TABLE `exchangis_launchable_task` ( + `id` bigint(13) NOT NULL, + `name` varchar(100) NOT NULL, + `job_execution_id` varchar(64) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime(3) DEFAULT NULL, + `engine_type` varchar(45) DEFAULT '', + `execute_user` varchar(50) DEFAULT '', + `linkis_job_name` varchar(100) NOT NULL, + `linkis_job_content` text NOT NULL, + `linkis_params` text DEFAULT NULL, + `linkis_source` varchar(64) DEFAULT NULL, + `labels` varchar(64) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- exchangis_launched_job_entity definition +DROP TABLE IF EXISTS `exchangis_launched_job_entity`; +CREATE TABLE `exchangis_launched_job_entity` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(100) NOT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime(3) DEFAULT NULL, + `job_id` bigint(20) DEFAULT NULL, + `launchable_task_num` int(20) DEFAULT '0', + `engine_type` varchar(100) DEFAULT NULL, + `execute_user` varchar(100) DEFAULT NULL, + `job_name` varchar(100) DEFAULT NULL, + `status` varchar(100) DEFAULT NULL, + `progress` varchar(100) DEFAULT NULL, + `error_code` varchar(64) DEFAULT NULL, + `error_msg` varchar(255) DEFAULT NULL, + `retry_num` bigint(10) DEFAULT NULL, + `job_execution_id` varchar(255) DEFAULT NULL, + `log_path` varchar(255) DEFAULT NULL, + `create_user` varchar(100) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `job_execution_id_UNIQUE` (`job_execution_id`) +) ENGINE=InnoDB AUTO_INCREMENT=8380 DEFAULT CHARSET=utf8; + +-- exchangis_launched_task_entity definition +DROP TABLE IF EXISTS `exchangis_launched_task_entity`; +CREATE TABLE `exchangis_launched_task_entity` ( + `id` bigint(20) NOT NULL, + `name` varchar(100) NOT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime(3) DEFAULT NULL, + `job_id` bigint(20) DEFAULT NULL, + `engine_type` varchar(100) DEFAULT NULL, + `execute_user` varchar(100) DEFAULT NULL, + `job_name` varchar(100) DEFAULT NULL, + `progress` varchar(64) DEFAULT NULL, + `error_code` varchar(64) DEFAULT NULL, + `error_msg` varchar(255) DEFAULT NULL, + `retry_num` bigint(10) DEFAULT NULL, + `task_id` varchar(64) DEFAULT NULL, + `linkis_job_id` varchar(200) DEFAULT NULL, + `linkis_job_info` varchar(1000) DEFAULT NULL, + `job_execution_id` varchar(100) DEFAULT NULL, + `launch_time` datetime DEFAULT NULL, + `running_time` datetime DEFAULT NULL, + `metrics` text, + `status` varchar(64) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- exchangis_job_func definition +DROP TABLE IF EXISTS `exchangis_job_func`; +CREATE TABLE `exchangis_job_func` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `func_type` varchar(50) NOT NULL, + `func_name` varchar(100) NOT NULL, + `tab_name` varchar(50) NOT NULL COMMENT 'Tab', + `name_dispaly` varchar(100) DEFAULT NULL, + `param_num` int(11) DEFAULT '0', + `ref_name` varchar(100) DEFAULT NULL, + `description` varchar(200) DEFAULT NULL, + `modify_time` datetime DEFAULT NULL, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `job_func_tab_name_idx` (`tab_name`,`func_name`) +) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8 + +-- exchangis_job_func_params definition +DROP TABLE IF EXISTS `exchangis_job_func_params`; +CREATE TABLE IF NOT EXISTS `exchangis_job_func_params`( + `func_id` INT(11) NOT NULL, + `param_name` VARCHAR(100) NOT NULL, + `order` INT(11) DEFAULT 0, + `name_display` VARCHAR(100), + `create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(`func_id`, `param_name`) +)Engine=InnoDB DEFAULT CHARSET=utf8; + +-- exchangis_engine_resources definition +DROP TABLE IF EXISTS `exchangis_engine_resources` +CREATE TABLE `exchangis_engine_resources` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `engine_type` varchar(50) NOT NULL, + `resource_name` varchar(100) NOT NULL, + `resource_type` varchar(50) NOT NULL COMMENT 'resource type' DEFAULT 'file', + `resource_path` varchar(255) NOT NULL, + `store_uri` varchar(500) NOT NULL, + `create_user` varchar(50) NOT NULL, + `modify_time` datetime DEFAULT NULL, + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `engine_res_idx` (`engine_type`,`resource_path`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- exchangis_engine_settings definition +DROP TABLE IF EXISTS `exchangis_engine_settings`; +CREATE TABLE `exchangis_engine_settings` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `engine_name` varchar(50) NOT NULL, + `engine_desc` varchar(500) NOT NULL, + `engine_settings_value` text, + `engine_direction` varchar(255) NOT NULL, + `res_loader_class` varchar(255), + `res_uploader_class` varchar(255), + `modify_time` datetime DEFAULT NULL, + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `engine_setting_idx` (`engine_name`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- exchangis_job_transform_rule +DROP TABLE IF EXISTS `exchangis_job_transform_rule`; +CREATE TABLE `exchangis_job_transform_rule` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `rule_name` varchar(100) NOT NULL DEFAULT 'transform_rule', + `rule_type` varchar(64) NOT NULL DEFAULT 'DEF', + `rule_source` varchar(600) DEFAULT '{}', + `data_source_type` varchar(64) NOT NULL, + `engine_type` varchar(32), + `direction` varchar(32) NOT NULL DEFAULT 'NONE', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- exchangis_job_transform_processor +DROP TABLE IF EXISTS `exchangis_job_transform_processor`; +CREATE TABLE `exchangis_job_transform_processor` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_id` bigint(20) NOT NULL, + `code_content` text DEFAULT NULL, + `code_language` varchar(32) NOT NULL DEFAULT 'java', + `code_bml_resourceId` varchar(255) COMMENT 'BML resource id', + `code_bml_version` varchar(255) COMMENT 'BML version', + `creator` varchar(50) NOT NULL COMMENT 'Owner of processor', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file diff --git a/db/1.1.1/exchangis_dml.sql b/db/1.1.1/exchangis_dml.sql new file mode 100644 index 000000000..a79efdf80 --- /dev/null +++ b/db/1.1.1/exchangis_dml.sql @@ -0,0 +1,81 @@ +-- job_func records +INSERT INTO exchangis_job_func (func_type,func_name,tab_name,name_dispaly,param_num,ref_name,description,modify_time,create_time) VALUES +('TRANSFORM','dx_substr','DATAX',NULL,2,NULL,NULL,NULL,'2020-04-21 17:35:06.000') +,('TRANSFORM','dx_pad','DATAX',NULL,3,NULL,NULL,NULL,'2020-04-21 17:35:06.000') +,('TRANSFORM','dx_replace','DATAX',NULL,3,NULL,NULL,NULL,'2020-04-21 17:35:06.000') +,('VERIFY','like','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') +,('VERIFY','not like','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') +,('VERIFY','>','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') +,('VERIFY','<','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') +,('VERIFY','=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') +,('VERIFY','!=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') +,('VERIFY','>=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') +; +INSERT INTO exchangis_job_func (func_type,func_name,tab_name,name_dispaly,param_num,ref_name,description,modify_time,create_time) VALUES +('TRANSFORM','dx_precision','DATAX',NULL,1,NULL,NULL,NULL,'2020-04-21 17:35:06.000') +; + +-- job_func_params records +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(1, 'startIndex', 'startIndex', 0) ON DUPLICATE KEY UPDATE `name_display` = 'startIndex'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(1, 'length', 'length', 1) ON DUPLICATE KEY UPDATE `name_display` = 'length'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(2, 'padType', 'padType(r or l)', 0) ON DUPLICATE KEY UPDATE `name_display` = 'padType(r or l)'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(2, 'length', 'length', 1) ON DUPLICATE KEY UPDATE `name_display` = 'length'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(2, 'padString', 'padString', 2) ON DUPLICATE KEY UPDATE `name_display` = 'padString'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(3, 'startIndex', 'startIndex', 0) ON DUPLICATE KEY UPDATE `name_display` = 'startIndex'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(3, 'length', 'length', 1) ON DUPLICATE KEY UPDATE `name_display` = 'length'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(3, 'replaceString', 'replaceString', 2) ON DUPLICATE KEY UPDATE `name_display` = 'replaceString'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(4, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(5, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(6, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(7, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(8, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(9, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(10, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; + +-- job_param_config records +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('setting.speed.byte','作业速率限制','','DATAX','INPUT','setting.speed.bytes','作业速率限制','Mb/s',1,'NUMBER','','5','REGEX','^[1-9]d*$','作业速率限制输入错误',0,0,'',1,'',1,'',1,NULL) +,('setting.speed.record','作业记录数限制','','DATAX','INPUT','setting.speed.records','作业记录数限制','条/s',1,'NUMBER','','100','REGEX','^[1-9]d*$','作业记录数限制输入错误',0,0,'',1,'',2,'',1,NULL) +,('setting.speed.channel','作业最大并行度','','DATAX','INPUT','setting.max.parallelism','作业最大并行度','个',1,'NUMBER','','1','REGEX','^[1-9]d*$','作业最大并行度输入错误',0,0,'',1,'',3,'',1,NULL) +,('setting.max.memory','作业最大使用内存','','DATAX','INPUT','setting.max.memory','作业最大使用内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]d*$','作业最大使用内存输入错误',0,0,'',1,'',4,'',1,NULL) +,('setting.errorLimit.record','最多错误记录数','','DATAX','INPUT','setting.errorlimit.record','最多错误记录数','条',0,'NUMBER','','','REGEX','^[0-9]d*$','最多错误记录数输入错误',0,0,'',1,'',5,'',1,NULL) +,('setting.max.parallelism','作业最大并行数','','SQOOP','INPUT','setting.max.parallelism','作业最大并行数','个',1,'NUMBER','','1','REGEX','^[1-9]d*$','作业最大并行数输入错误',0,0,'',1,'',1,'',1,NULL) +,('setting.max.memory','作业最大内存','','SQOOP','INPUT','setting.max.memory','作业最大内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]d*$','作业最大内存输入错误',0,0,'',1,'',2,'',1,NULL) +,('where','WHERE条件','SOURCE','MYSQL','INPUT','where','WHERE条件','',0,'VARCHAR','','','REGEX','^[sS]{0,500}$','WHERE条件输入过长',0,0,'',1,'',2,'',1,NULL) +,('writeMode','写入方式','SQOOP-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["OVERWRITE","APPEND"]','OVERWRITE','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) +; +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) +,('writeMode','写入方式','SQOOP-SINK','MYSQL','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","UPDATE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('batchSize','批量大小','DATAX-SINK','ELASTICSEARCH','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]d*$','批量大小输入错误',0,0,'',1,'',1,'',1,NULL) +,('query','query条件','DATAX-SOURCE','MONGODB','INPUT','query','query条件','',0,'VARCHAR','','','REGEX','^[sS]{0,500}$','query条件输入过长',0,0,'',1,'',2,'',1,NULL) +,('writeMode','写入方式','DATAX-SINK','MONGODB','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","REPLACE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('batchSize','批量大小','DATAX-SINK','MONGODB','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL) +,('transferMode','传输方式','DATAX-SOURCE','HIVE','OPTION','transferMode','传输方式','',1,'OPTION','["二进制","记录"]','二进制','','','该传输方式不可用',0,0,'',1,'',1,'',1,NULL) +,('nullFormat','空值字符','DATAX-SOURCE','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +,('writeMode','写入方式','DATAX-SINK','MYSQL','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","UPDATE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('writeMode','写入方式','DATAX-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["append","truncate"]','append','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +; +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('nullFormat','空值字符','DATAX-SINK','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +,('nullFormat','空值字符','DATAX-SINK','ELASTICSEARCH','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +; + +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('where','WHERE条件','SOURCE','ORACLE','INPUT','where','WHERE条件',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,NULL,1,'',2,NULL,1,NULL) +,('writeMode','写入方式','DATAX-SINK','ORACLE','OPTION','writeMode','写入方式',NULL,1,'OPTION','["INSERT","UPDATE"]','INSERT',NULL,NULL,'写入方式输入错误',0,0,NULL,1,NULL,1,NULL,1,NULL) +; + +-- engine_settings records +INSERT INTO `exchangis_engine_settings` (id, engine_name, engine_desc, engine_settings_value, engine_direction, res_loader_class, res_uploader_class, modify_time, create_time) VALUES +(1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL, '2022-08-09 18:20:51.0'), +(2, 'sqoop', 'hadoop tool', '{}', 'mysql->hive,hive->mysql', '', NULL, NULL, '2022-08-09 18:20:51.0'); + +INSERT INTO `exchangis_job_transform_rule` (rule_name,rule_type,rule_source,data_source_type,engine_type,direction,create_time) VALUES +('es_with_post_processor','DEF','{"types": ["MAPPING", "PROCESSOR"]}','ELASTICSEARCH',NULL,'SINK','2022-09-18 23:36:42.000') +,('es_fields_not_editable','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false}','ELASTICSEARCH',NULL,'SINK','2022-09-20 16:27:18.000') +,('hive_sink_not_access','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false, "fieldAddEnable": false}','HIVE',NULL,'SINK','2022-09-20 21:46:05.000') +,('mongo_field_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH"}','MONGODB',NULL,'SINK','2022-09-20 21:55:05.000') +,('mysql_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": false}','MYSQL',NULL,'SOURCE','2022-09-21 21:55:05.000') +; \ No newline at end of file diff --git a/db/1.1.2/exchangis_ddl.sql b/db/1.1.2/exchangis_ddl.sql new file mode 100644 index 000000000..3609cadfd --- /dev/null +++ b/db/1.1.2/exchangis_ddl.sql @@ -0,0 +1 @@ +ALTER TABLE exchangis_job_entity MODIFY COLUMN name varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL; \ No newline at end of file diff --git a/db/exchangis_ddl.sql b/db/exchangis_ddl.sql index 58f1e1df2..c04796624 100644 --- a/db/exchangis_ddl.sql +++ b/db/exchangis_ddl.sql @@ -14,7 +14,7 @@ CREATE TABLE `exchangis_job_ds_bind` ( DROP TABLE IF EXISTS `exchangis_job_entity`; CREATE TABLE `exchangis_job_entity` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, - `name` varchar(100) NOT NULL, + `name` varchar(255) NOT NULL, `create_time` datetime DEFAULT NULL, `last_update_time` datetime(3) DEFAULT NULL, `engine_type` varchar(45) DEFAULT '', @@ -178,7 +178,7 @@ CREATE TABLE `exchangis_job_func` ( `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `job_func_tab_name_idx` (`tab_name`,`func_name`) -) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8 +) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8; -- exchangis_job_func_params definition DROP TABLE IF EXISTS `exchangis_job_func_params`; diff --git a/docs/en_US/ch1/component_upgrade_en.md b/docs/en_US/ch1/component_upgrade_en.md index 7f9e69d95..c3aa23379 100644 --- a/docs/en_US/ch1/component_upgrade_en.md +++ b/docs/en_US/ch1/component_upgrade_en.md @@ -1,7 +1,7 @@ # Exchangis Component Upgrade Documentation -This article mainly introduces the upgrade steps for adapting DSS1.1.0 and Linkis1.1.1 on the basis of the original installation of the Exchangis service. The biggest difference between the Exchangis1.0.0 and the Exchangis1.0.0-rc1 version is the installation of the ExchangisAppconn, which needs to be replaced by the entire Exchangisappconn. and load### 1.升级Exchangis前的工作 -Before you upgrade Exchangis, please follow the[DSS1.1.0Install and deploy documentation](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_deploy_cn.md) -and [Linkis1.1.1Install and deploy documentation](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_deploy_cn.md)Complete the installation and upgrade of DSS and Linkis +This article mainly introduces the upgrade steps for adapting DSS1.1.2 and Linkis1.4.0 on the basis of the original installation of the Exchangis service. The biggest difference between the Exchangis1.0.0 and the Exchangis1.0.0-rc1 version is the installation of the ExchangisAppconn, which needs to be replaced by the entire Exchangisappconn. and load### 1.升级Exchangis前的工作 +Before you upgrade Exchangis, please follow the[DSS1.1.2Install and deploy documentation](https://github.com/WeBankFinTech/DataSphereStudio-Doc/tree/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2) +and [Linkis1.4.0Install and deploy documentation](https://linkis.staged.apache.org/zh-CN/docs/1.4.0/deployment/deploy-quick)Complete the installation and upgrade of DSS and Linkis ### 2.Exchangis upgrade steps diff --git a/docs/en_US/ch1/exchangis_datasource_en.md b/docs/en_US/ch1/exchangis_datasource_en.md index 9dc125718..ffbffdf98 100644 --- a/docs/en_US/ch1/exchangis_datasource_en.md +++ b/docs/en_US/ch1/exchangis_datasource_en.md @@ -106,7 +106,7 @@ Batch import and export of datasource configuration. Description: -The background needs to integrate it with the labeling function of Linkis1.1.1, and give the datasource a labeling relationship. +The background needs to integrate it with the labeling function of Linkis1.4.0, and give the datasource a labeling relationship. Process design: diff --git a/docs/en_US/ch1/exchangis_deploy_en.md b/docs/en_US/ch1/exchangis_deploy_en.md index faecef887..c80d02484 100644 --- a/docs/en_US/ch1/exchangis_deploy_en.md +++ b/docs/en_US/ch1/exchangis_deploy_en.md @@ -18,17 +18,17 @@ Exchangis installation is mainly divided into the following four steps ： | Hadoop(2.7.2，Other versions of Hadoop need to compile Linkis by themselves.) | yes | [Hadoop stand-alone deployment](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) ；[Hadoop distributed deployment](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | | Hive(2.3.3，Other versions of Hive need to compile Linkis by themselves.) | yes | [Hive quick installation](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | | SQOOP (1.4.6) | yes | [How to install Sqoop](https://sqoop.apache.org/docs/1.4.6/SqoopUserGuide.html) | -| DSS1.1.0 | yes | [How to install DSS](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) | -| Linkis1.1.1 | yes | [How to install Linkis](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | +| DSS1.1.2 | yes | [How to install DSS](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) | +| Linkis1.4.0 | yes | [How to install Linkis](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | | Nginx | yes | [How to install Nginx](http://nginx.org/en/linux_packages.html) | Underlying component checking -$\color{#FF0000}{Note: be sure to reinstall dss1.1.0, and the linkis version must be greater than 1.1.1. Please recompile linkis and use the package released on June 15th }$ +$\color{#FF0000}{Note: be sure to reinstall dss1.1.2, and linkis1.4.0. Please recompile linkis and use the package released on June 15th }$ -[linkis1.1.1 code address ](https://github.com/apache/incubator-linkis/tree/release-1.1.1) +[linkis1.4.0 code address ](https://github.com/apache/incubator-linkis/tree/release-1.4.0) -[DSS1.1.0 code address ](https://github.com/WeBankFinTech/DataSphereStudio/tree/dev-1.1.0) +[DSS1.1.2 code address ](https://github.com/WeBankFinTech/DataSphereStudio/tree/dev-1.1.2) datasource enabled @@ -55,11 +55,11 @@ INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_t INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_type_id`, `parameter`, `create_time`, `create_user`, `modify_time`, `modify_user`) VALUES ('开发环境UAT', '开发环境UAT', 4, '{"uris":"thrift://${HIVE_METADATA_IP}:${HIVE_METADATA_PORT}", "hadoopConf":{"hive.metastore.execute.setugi":"true"}}', now(), NULL, now(), NULL); ``` -If the hive data source needs kerberos authentication when deployed, you need to specify a parameter keyTab in the parameter field of the Linkis_ps_dm_datasource_env table, and the way to obtain its value can be seen: [Setting and authenticating hive data source in linkis](https://linkis.apache.org/zh-CN/docs/1.1.1/deployment/start-metadatasource). +If the hive data source needs kerberos authentication when deployed, you need to specify a parameter keyTab in the parameter field of the Linkis_ps_dm_datasource_env table, and the way to obtain its value can be seen: [Setting and authenticating hive data source in linkis](https://linkis.apache.org/zh-CN/docs/1.4.0/deployment/start-metadatasource). #### 1.4 Underlying component checking -Please ensure that DSS1.1.0 and Linkis1.1.1 are basically available. HiveQL scripts can be executed in the front-end interface of DSS, and DSS workflows can be created and executed normally. +Please ensure that DSS1.1.2 and Linkis1.4.0 are basically available. HiveQL scripts can be executed in the front-end interface of DSS, and DSS workflows can be created and executed normally. ## 2. Exchangis installation and deployment diff --git a/docs/en_US/ch1/exchangis_sqoop_deploy_en.md b/docs/en_US/ch1/exchangis_sqoop_deploy_en.md index 60f796121..b84e3f7cf 100644 --- a/docs/en_US/ch1/exchangis_sqoop_deploy_en.md +++ b/docs/en_US/ch1/exchangis_sqoop_deploy_en.md @@ -26,7 +26,7 @@ It is strongly recommended that you use the native Sqoop to perform the test tas ### Prepare installation package #### 1）Download binary package -Exchangis1.0.0 and Linkis 1.1.1 support the mainstream Sqoop versions 1.4.6 and 1.4.7, and later versions may need to modify some codes for recompilation. +Exchangis1.1.2 and Linkis 1.4.0 support the mainstream Sqoop versions 1.4.6 and 1.4.7, and later versions may need to modify some codes for recompilation. [Click to jump to Release interface](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.0.0) diff --git a/docs/zh_CN/ch1/component_upgrade_cn.md b/docs/zh_CN/ch1/component_upgrade_cn.md index fd4501f35..7858b31b3 100644 --- a/docs/zh_CN/ch1/component_upgrade_cn.md +++ b/docs/zh_CN/ch1/component_upgrade_cn.md @@ -1,8 +1,8 @@ Exchangis 升级文档 -本文主要介绍在原有安装Exchangis服务的基础上适配DSS1.1.0和Linkis1.1.1的升级步骤，Exchangis1.0.0相对与Exchangis1.0.0-rc1版本最大的区别在于ExchangisAppconn的安装，需要对整个Exchangisappconn进行重新替换和加载 +本文主要介绍在原有安装Exchangis服务的基础上适配DSS1.1.2和Linkis1.4.0的升级步骤，Exchangis1.1.2相对与Exchangis1.0.0-rc1版本最大的区别在于ExchangisAppconn的安装，需要对整个Exchangisappconn进行重新替换和加载 ### 1.升级Exchangis前的工作 -您在升级Exchangis之前，请按照[DSS1.1.0安装部署文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_deploy_cn.md) -和[Linkis1.1.1安装部署文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_deploy_cn.md)完成DSS和Linkis的安装升级 +您在升级Exchangis之前，请按照[DSS1.1.2安装部署文档](https://github.com/WeBankFinTech/DataSphereStudio-Doc/tree/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2) +和[Linkis1.4.0安装部署文档](https://linkis.staged.apache.org/zh-CN/docs/1.4.0/deployment/deploy-quick)完成DSS和Linkis的安装升级 ### 2.Exchangis升级步骤 @@ -14,7 +14,7 @@ Exchangis 升级文档 ``` #### 2）下载二进制包 -我们提供ExchangisAppconn的升级物料包，您可直接下载使用。[点击跳转 Release 界面](https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeDatasphere/Exchangis/exchangis1.0.0-rc/exchangis-appconn.zip) +我们提供ExchangisAppconn的升级物料包，您可直接下载使用。[点击跳转 Release 界面](https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeDatasphere/Exchangis/exchangis1.0.0-rc/exchangis-appconn.zip)（TODO：待发版时更新） #### 3） 编译打包 @@ -73,7 +73,7 @@ lib/exchangis-server ``` 再通过以下命令完成 Exchangis Server 的更新重启： - + ```shell script ./sbin/daemon.sh restart server ``` @@ -94,4 +94,4 @@ wds.linkis-session.ticket.key=bdp-user-ticket-id ![image](https://user-images.githubusercontent.com/27387830/169782337-678f2df0-080a-495a-b59f-a98c5a427cf8.png) - 更多使用操作可参照[Exchangis1.0用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_user_manual_cn.md) + 更多使用操作可参照[Exchangis用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.1.2/docs/zh_CN/ch1/exchangis_user_manual_cn.md) diff --git a/docs/zh_CN/ch1/exchangis_deploy_cn.md b/docs/zh_CN/ch1/exchangis_deploy_cn.md index 6c7376e96..2ec1d41e1 100644 --- a/docs/zh_CN/ch1/exchangis_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_deploy_cn.md @@ -19,21 +19,21 @@ Exchangis 的安装，主要分为以下四步： | Hadoop(2.7.2，Hadoop 其他版本需自行编译 Linkis) | 必装 | [Hadoop单机部署](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) ；[Hadoop分布式部署](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | | Hive(2.3.3，Hive 其他版本需自行编译 Linkis) | 必装 | [Hive快速安装](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | | SQOOP (1.4.6) | 必装 | [如何安装Sqoop](https://sqoop.apache.org/docs/1.4.6/SqoopUserGuide.html) | -| DSS1.1.0 | 必装 | [如何安装DSS](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) | -| Linkis1.1.1 | 必装 | [如何安装Linkis](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | +| DSS1.1.2 | 必装 | [如何安装DSS](https://github.com/WeBankFinTech/DataSphereStudio-Doc/tree/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2) | +| Linkis1.4.0 | 必装 | [如何安装Linkis](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | | Nginx | 必装 | [如何安装 Nginx](http://nginx.org/en/linux_packages.html) | 底层依赖组件检查 -$\color{#FF0000}{注意：一定要使用最新版的dss1.1.0，及linkis1.1.1}$。 +注意：一定要使用最新版的dss1.1.2，及linkis1.4.0。 -[linkis1.1.1代码地址](https://github.com/apache/incubator-linkis/tree/release-1.1.1) +[linkis1.4.0代码地址](https://github.com/apache/incubator-linkis/tree/release-1.4.0) -[DSS1.1.0代码地址](https://github.com/WeBankFinTech/DataSphereStudio/tree/dev-1.1.0) +[DSS1.1.2代码地址 ](https://github.com/WeBankFinTech/DataSphereStudio/tree/dev-1.1.2) datasource启用 -linkis的启动脚本中默认不会启动数据源相关的服务两个服务（ps-data-source-manager，ps-metadatamanager）， 如果想使用数据源服务，可以通过如下方式进行开启: 修改$LINKIS_CONF_DIR/linkis-env.sh中的 export ENABLE_METADATA_MANAGER=true值为true。 通过linkis-start-all.sh/linkis-stop-all.sh 进行服务启停时，会进行数据源服务的启动与停止。关于数据源更多详情可参考[数据源功能使用](https://linkis.apache.org/zh-CN/docs/1.1.0/deployment/start-metadatasource) +linkis的启动脚本中默认不会启动数据源相关的服务两个服务（ps-data-source-manager，ps-metadatamanager）， 如果想使用数据源服务，可以通过如下方式进行开启: 修改$LINKIS_CONF_DIR/linkis-env.sh中的 export ENABLE_METADATA_MANAGER=true值为true。 通过linkis-start-all.sh/linkis-stop-all.sh 进行服务启停时，会进行数据源服务的启动与停止。关于数据源更多详情可参考[数据源功能使用](https://linkis.apache.org/zh-CN/docs/1.4.0/user-guide/datasource-manual) #### 1.2 创建 Linux 用户 @@ -58,11 +58,11 @@ INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_t INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_type_id`, `parameter`, `create_time`, `create_user`, `modify_time`, `modify_user`) VALUES ('开发环境UAT', '开发环境UAT', 4, '{"uris":"thrift://${HIVE_METADATA_IP}:${HIVE_METADATA_PORT}", "hadoopConf":{"hive.metastore.execute.setugi":"true"}}', now(), NULL, now(), NULL); ``` -如果hive数据源在部署时设置了需要进行kerberos方式认证，则需要在linkis_ps_dm_datasource_env表的parameter字段指定一个参数keyTab，其值的获取方式可见：[在Linkis中设置并认证hive数据源](https://linkis.apache.org/zh-CN/docs/1.1.1/deployment/start-metadatasource) +如果hive数据源在部署时设置了需要进行kerberos方式认证，则需要在linkis_ps_dm_datasource_env表的parameter字段指定一个参数keyTab，其值的获取方式可见：[在Linkis中设置并认证hive数据源](https://linkis.apache.org/zh-CN/docs/1.4.0/deployment/start-metadatasource) #### 1.4 底层依赖组件检查 -**请确保 DSS1.1.0 与 Linkis1.1.1 基本可用，可在 DSS 前端界面执行 HiveQL 脚本，可正常创建并执行 DSS 工作流。** +**请确保 DSS1.1.2 与 Linkis1.4.0 基本可用，可在 DSS 前端界面执行 HiveQL 脚本，可正常创建并执行 DSS 工作流。** ## 2. Exchangis 安装部署 @@ -71,7 +71,7 @@ INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_t #### 2.1.1下载二进制包 -从 Exchangis 已发布的 release 中 [点击下载exchangis安装包](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.0.0)，下载最新的安装包。 +从 Exchangis 已发布的 release 中 [点击下载exchangis安装包](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.1.2)，下载最新的安装包。 #### 2.1.2 编译打包 @@ -157,7 +157,27 @@ DATABASE={dbName} 输入 `y` 初始化数据库表，输入 `n` 跳过数据库表初始化步骤。 -#### 2.5.3 启动服务 +#### 2.5.3 修改配置文件路径和日志文件路径 + +在sbin目录下的`env.properties`文件，设置配置文件路径和日志文件路径 + +```yaml +EXCHANGIS_CONF_PATH="/appcom/config/exchangis-config/background" +EXCHANGIS_LOG_PATH="/appcom/logs/exchangis-log" +MODULE_DEFAULT_PREFIX="exchangis-server" +MODULE_DEFAULT_SUFFIX="" +``` + +EXCHANGIS_CONF_PATH为配置文件路径，EXCHANGIS_LOG_PATH为日志文件路径，若为以上配置，则作如下操作： + +```shell +cp config/* /appcom/config/exchangis-config/background +mkdir -p /appcom/logs/exchangis-log +``` + +则在服务启动时，将会使用对应路径下的配置文件，以及将日志写到对应的路径下 + +#### 2.5.4 启动服务 第一次启动，可以sbin目录下执行以下命令，启动 Exchangis Server： @@ -189,7 +209,7 @@ DATABASE={dbName} #### 2.7.1 获取前端安装包 -Exchangis 已默认提供了编译好的前端安装包，可直接下载使用：[点击下载前端安装包](https://github.com/WeBankFinTech/Exchangis/releases/download/release-1.0.0-rc1/web-dist.zip) +Exchangis 已默认提供了编译好的前端安装包，可直接下载使用：[点击下载前端安装包](https://github.com/WeBankFinTech/Exchangis/releases/download/release-1.1.2/web-dist.zip) 您也可以自行编译 Exchangis 前端，在 Exchangis 根目录下执行如下命令： @@ -199,7 +219,7 @@ Exchangis 已默认提供了编译好的前端安装包，可直接下载使用 npm run build ``` -从 `web/` 路径获取编译好的 dist.zip 前端包。 +从 `web/` 路径获取编译好的 exchangis-ui.zip 前端包。 获取到的前端包，您可以放在服务器上的任意位置，这里建议您与后端安装地址目录保持一致，在同一目录下放置并解压。 @@ -207,15 +227,14 @@ Exchangis 已默认提供了编译好的前端安装包，可直接下载使用 1. 解压前端安装包 -如您打算将 Exchangis 前端包部署到 `/appcom/Install/exchangis/web` 目录，请先将 `dist.zip` 拷贝到该目录并执行解压，注意，**请在安装dss的机器上安装exchangis前端**： +如您打算将 Exchangis 前端包部署到 `/appcom/Install/ExchangisInstall/exchangis-ui` 目录，请先将 `exchangis-ui.zip` 拷贝到该目录并执行解压，注意，**请在安装dss的机器上安装exchangis前端**： ```shell script - # 请先将 Exchangis 前端包拷贝到 `/appcom/Install/exchangis/web` 目录 - cd /appcom/Install/exchangis/web - unzip dist.zip + # 请先将 Exchangis 前端包拷贝到 `/appcom/Install/ExchangisInstall` 目录 + cd /appcom/Install/ExchangisInstall + unzip exchangis-ui.zip ``` - 执行如下命令： ```shell script @@ -228,8 +247,8 @@ Exchangis 已默认提供了编译好的前端安装包，可直接下载使用 server_name localhost; #charset koi8-r; #access_log /var/log/nginx/host.access.log main; - location /dist { - root /appcom/Install/exchangis/web; # Exchangis 前端部署目录 + location / { + root /appcom/Install/ExchangisInstall/exchangis-ui; # Exchangis 前端部署目录 autoindex on; } @@ -272,16 +291,16 @@ Exchangis 已默认提供了编译好的前端安装包，可直接下载使用 ## 3. DSS ExchangisAppConn 安装部署 -如您想正常使用 Exchangis1.0.0 前端，还需安装 DSS ExchangisAppConn 插件，请参考: [ExchangisAppConn 插件安装文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) +如您想正常使用 Exchangis1.1.2 前端，还需安装 DSS ExchangisAppConn 插件，请参考: [ExchangisAppConn 插件安装文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.1.2/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) ## 4. Linkis Sqoop 引擎安装部署 -如您想正常执行 Exchangis1.0.0 的 Sqoop作业，还需安装 Linkis Sqoop 引擎，请参考: [Linkis Sqoop 引擎插件安装文档](https://linkis.staged.apache.org/zh-CN/docs/1.1.2/engine_usage/sqoop) +如您想正常执行 Exchangis1.1.2 的 Sqoop作业，还需安装 Linkis Sqoop 引擎，请参考: [Linkis Sqoop 引擎插件安装文档](https://linkis.staged.apache.org/zh-CN/docs/1.4.0/engine_usage/sqoop) ## 5. Linkis DataX 引擎安装部署 -如您想正常执行 Exchangis1.0.0 的 DataX作业，还需安装 Linkis DataX 引擎，请参考: [Linkis DataX 引擎插件安装文档](https://linkis.staged.apache.org/zh-CN/docs/1.1.2/engine_usage/datax)（TODO：待Linkis确认） +如您想正常执行 Exchangis1.0.0 的 DataX作业，还需安装 Linkis DataX 引擎，请参考: [Linkis DataX 引擎插件安装文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.1.2/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md) ## 6. 如何登录使用 Exchangis -Exchangis1.0更多使用说明，请参考用户使用手册[Exchangis1.0 用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_user_manual_cn.md) +Exchangis更多使用说明，请参考用户使用手册[Exchangis 用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.1.2/docs/zh_CN/ch1/exchangis_user_manual_cn.md) diff --git a/exchangis-plugins/exchangis-appconn/src/main/resources/datax.icon b/exchangis-plugins/exchangis-appconn/src/main/resources/datax.icon new file mode 100644 index 000000000..bf047fb7b --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/resources/datax.icon @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql index 97a567232..55105578c 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql +++ b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql @@ -6,7 +6,7 @@ delete from `dss_workspace_menu_appconn` where `appconn_id` in (select `id` fro delete from `dss_appconn` where `appconn_name`='exchangis'; INSERT INTO `dss_appconn` (`appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, `reference`, `class_name`, `appconn_class_path`, `resource`) -VALUES ('exchangis', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.exchangis.dss.appconn.ExchangisAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/exchangis/lib', ''); +VALUES ('exchangis', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.exchangis.dss.appconn.ExchangisAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/exchangis', ''); select @dss_appconn_exchangis_id:=id from `dss_appconn` where `appconn_name` = "exchangis"; INSERT INTO `dss_appconn_instance` (`appconn_id`, `label`, `url`, `enhance_json`, `homepage_uri`) @@ -30,7 +30,7 @@ insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type values('sqoop','exchangis','linkis.appconn.exchangis.sqoop',1,'1','1','0','1','icons/sqoop.icon'); select @dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'; insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) -values('exchangis','exchangis','linkis.appconn.exchangis.datax',1,'1','1','0','1','icons/datax.icon'); +values('datax','exchangis','linkis.appconn.exchangis.datax',1,'1','1','0','1','icons/datax.icon'); select @dss_exchangis_dataxId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'; -- 节点组表dss_workflow_node_to_group diff --git a/exchangis-server/src/main/resources/exchangis.properties b/exchangis-server/src/main/resources/exchangis.properties index 0b906a666..4be93901f 100644 --- a/exchangis-server/src/main/resources/exchangis.properties +++ b/exchangis-server/src/main/resources/exchangis.properties @@ -34,7 +34,7 @@ wds.exchangis.datasource.client.authtoken.key=hdfs wds.exchangis.datasource.client.authtoken.value=exchangis-auth wds.exchangis.datasource.client.dws.version=v1 -wds.exchangis.datasource.extension.dir=exchangis-extds +wds.exchangis.datasource.extension.dir=exchangis-extds/ ##restful wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.exchangis.datasource.server.restful.api,\ From 29006bb5c2afe1e647e3c0633e24c4904d02fad3 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Tue, 18 Jul 2023 21:56:40 +0800 Subject: [PATCH 13/20] Remove the letter v from the version number when packaging the engine --- .../engineconn-plugins/datax/src/main/assembly/distribution.xml | 2 +- .../engineconn-plugins/sqoop/src/main/assembly/distribution.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml b/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml index 667d0f544..6ac8d1a5c 100644 --- a/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml +++ b/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml @@ -33,7 +33,7 @@ - /dist/v${datax.version}/lib + /dist/${datax.version}/lib true true false diff --git a/exchangis-engines/engineconn-plugins/sqoop/src/main/assembly/distribution.xml b/exchangis-engines/engineconn-plugins/sqoop/src/main/assembly/distribution.xml index 78f54c4c0..a88001e18 100644 --- a/exchangis-engines/engineconn-plugins/sqoop/src/main/assembly/distribution.xml +++ b/exchangis-engines/engineconn-plugins/sqoop/src/main/assembly/distribution.xml @@ -32,7 +32,7 @@ - /dist/v${sqoop.version}/lib + /dist/${sqoop.version}/lib true true false From 0bff39b3c93e9316d4046ccae2df62149396b1f3 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Tue, 18 Jul 2023 21:57:50 +0800 Subject: [PATCH 14/20] Update docs;Update the interface of deleteProject --- db/1.1.1/exchangis_ddl.sql | 2 +- db/exchangis_dml.sql | 41 ++++++++----------- docs/zh_CN/ch1/exchangis_datax_deploy_cn.md | 14 +++++-- docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md | 18 +++++--- .../exchangis/job/vo/ExchangisJobQueryVo.java | 7 ++++ .../restful/ExchangisProjectRestfulApi.java | 7 ++-- 6 files changed, 51 insertions(+), 38 deletions(-) diff --git a/db/1.1.1/exchangis_ddl.sql b/db/1.1.1/exchangis_ddl.sql index 766a4eb37..25906a604 100644 --- a/db/1.1.1/exchangis_ddl.sql +++ b/db/1.1.1/exchangis_ddl.sql @@ -178,7 +178,7 @@ CREATE TABLE `exchangis_job_func` ( `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `job_func_tab_name_idx` (`tab_name`,`func_name`) -) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8 +) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8; -- exchangis_job_func_params definition DROP TABLE IF EXISTS `exchangis_job_func_params`; diff --git a/db/exchangis_dml.sql b/db/exchangis_dml.sql index 1761fc734..57a38dc5d 100644 --- a/db/exchangis_dml.sql +++ b/db/exchangis_dml.sql @@ -10,9 +10,7 @@ INSERT INTO exchangis_job_func (func_type,func_name,tab_name,name_dispaly,param_ ,('VERIFY','=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') ,('VERIFY','!=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') ,('VERIFY','>=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') -; -INSERT INTO exchangis_job_func (func_type,func_name,tab_name,name_dispaly,param_num,ref_name,description,modify_time,create_time) VALUES -('TRANSFORM','dx_precision','DATAX',NULL,1,NULL,NULL,NULL,'2020-04-21 17:35:06.000') +,('TRANSFORM','dx_precision','DATAX',NULL,1,NULL,NULL,NULL,'2020-04-21 17:35:06.000') ; -- job_func_params records @@ -34,36 +32,33 @@ INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) -- job_param_config records INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES -('setting.speed.byte','作业速率限制','','DATAX','INPUT','setting.speed.bytes','作业速率限制','Mb/s',1,'NUMBER','','5','REGEX','^[1-9]d*$','作业速率限制输入错误',0,0,'',1,'',1,'',1,NULL) -,('setting.speed.record','作业记录数限制','','DATAX','INPUT','setting.speed.records','作业记录数限制','条/s',1,'NUMBER','','100','REGEX','^[1-9]d*$','作业记录数限制输入错误',0,0,'',1,'',2,'',1,NULL) -,('setting.speed.channel','作业最大并行度','','DATAX','INPUT','setting.max.parallelism','作业最大并行度','个',1,'NUMBER','','1','REGEX','^[1-9]d*$','作业最大并行度输入错误',0,0,'',1,'',3,'',1,NULL) -,('setting.max.memory','作业最大使用内存','','DATAX','INPUT','setting.max.memory','作业最大使用内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]d*$','作业最大使用内存输入错误',0,0,'',1,'',4,'',1,NULL) -,('setting.errorLimit.record','最多错误记录数','','DATAX','INPUT','setting.errorlimit.record','最多错误记录数','条',0,'NUMBER','','','REGEX','^[0-9]d*$','最多错误记录数输入错误',0,0,'',1,'',5,'',1,NULL) -,('setting.max.parallelism','作业最大并行数','','SQOOP','INPUT','setting.max.parallelism','作业最大并行数','个',1,'NUMBER','','1','REGEX','^[1-9]\d*$','作业最大并行数输入错误',0,0,'',1,'',1,'',1,NULL) -,('setting.max.memory','作业最大内存','','SQOOP','INPUT','setting.max.memory','作业最大内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\d*$','作业最大内存输入错误',0,0,'',1,'',2,'',1,NULL) -,('where','WHERE条件','SOURCE','MYSQL','INPUT','where','WHERE条件','',0,'VARCHAR','','','REGEX','^[sS]{0,500}$','WHERE条件输入过长',0,0,'',1,'',2,'',1,NULL) +('setting.speed.byte','作业速率限制','','DATAX','INPUT','setting.speed.bytes','作业速率限制','Mb/s',1,'NUMBER','','5','REGEX','^[1-9]\\d*$','作业速率限制输入错误',0,0,'',1,'',1,'',1,NULL) +,('setting.speed.record','作业记录数限制','','DATAX','INPUT','setting.speed.records','作业记录数限制','条/s',1,'NUMBER','','100','REGEX','^[1-9]\\d*$','作业记录数限制输入错误',0,0,'',1,'',2,'',1,NULL) +,('setting.speed.channel','作业最大并行度','','DATAX','INPUT','setting.max.parallelism','作业最大并行度','个',1,'NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行度输入错误',0,0,'',1,'',3,'',1,NULL) +,('setting.max.memory','作业最大使用内存','','DATAX','INPUT','setting.max.memory','作业最大使用内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大使用内存输入错误',0,0,'',1,'',4,'',1,NULL) +,('setting.errorLimit.record','最多错误记录数','','DATAX','INPUT','setting.errorlimit.record','最多错误记录数','条',0,'NUMBER','','','REGEX','^[0-9]\\d*$','最多错误记录数输入错误',0,0,'',1,'',5,'',1,NULL) +,('setting.max.parallelism','作业最大并行数','','SQOOP','INPUT','setting.max.parallelism','作业最大并行数','个',1,'NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行数输入错误',0,0,'',1,'',1,'',1,NULL) +,('setting.max.memory','作业最大内存','','SQOOP','INPUT','setting.max.memory','作业最大内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大内存输入错误',0,0,'',1,'',2,'',1,NULL) +,('where','WHERE条件','SOURCE','MYSQL','INPUT','where','WHERE条件','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,'',1,'',2,'',1,NULL) ,('writeMode','写入方式','SQOOP-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["OVERWRITE","APPEND"]','OVERWRITE','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) -,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) +,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) ; INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES -('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) +('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) ,('writeMode','写入方式','SQOOP-SINK','MYSQL','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","UPDATE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) -,('batchSize','批量大小','DATAX-SINK','ELASTICSEARCH','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\d*$','批量大小输入错误',0,0,'',1,'',1,'',1,NULL) -,('query','query条件','DATAX-SOURCE','MONGODB','INPUT','query','query条件','',0,'VARCHAR','','','REGEX','^[sS]{0,500}$','query条件输入过长',0,0,'',1,'',2,'',1,NULL) +,('batchSize','批量大小','DATAX-SINK','ELASTICSEARCH','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',1,'',1,NULL) +,('query','query条件','DATAX-SOURCE','MONGODB','INPUT','query','query条件','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,500}$','query条件输入过长',0,0,'',1,'',2,'',1,NULL) ,('writeMode','写入方式','DATAX-SINK','MONGODB','OPTION','writeMode','写入方式','',1,'OPTION','["insert","replace"]','insert','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) -,('batchSize','批量大小','DATAX-SINK','MONGODB','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL) +,('batchSize','批量大小','DATAX-SINK','MONGODB','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL) ,('transferMode','传输方式','DATAX-SOURCE','HIVE','OPTION','transferMode','传输方式','',1,'OPTION','["二进制","记录"]','二进制','','','该传输方式不可用',0,0,'',1,'',1,'',1,NULL) -,('nullFormat','空值字符','DATAX-SOURCE','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +,('nullFormat','空值字符','DATAX-SOURCE','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) ,('writeMode','写入方式','DATAX-SINK','MYSQL','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","UPDATE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) ,('writeMode','写入方式','DATAX-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["append","truncate"]','append','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) ; INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES -('nullFormat','空值字符','DATAX-SINK','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) -,('nullFormat','空值字符','DATAX-SINK','ELASTICSEARCH','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) -; - -INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES -('where','WHERE条件','SOURCE','ORACLE','INPUT','where','WHERE条件',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,NULL,1,'',2,NULL,1,NULL) +('nullFormat','空值字符','DATAX-SINK','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +,('nullFormat','空值字符','DATAX-SINK','ELASTICSEARCH','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +,('where','WHERE条件','SOURCE','ORACLE','INPUT','where','WHERE条件',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,NULL,1,'',2,NULL,1,NULL) ,('writeMode','写入方式','DATAX-SINK','ORACLE','OPTION','writeMode','写入方式',NULL,1,'OPTION','["INSERT","UPDATE"]','INSERT',NULL,NULL,'写入方式输入错误',0,0,NULL,1,NULL,1,NULL,1,NULL) ; diff --git a/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md b/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md index 250384a89..df854bd42 100644 --- a/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md @@ -27,12 +27,12 @@ Exchangis1.1.2和Linkis 1.4.0支持的主流DataX版本1.4.6与1.4.7，更高版 2、在exchangis-plugins模块下，找到datax引擎，单独编译datax，操作如下 ``` -cd {EXCHANGIS_CODE_HOME}/exchangis-plugins/engine/datax +cd {EXCHANGIS_CODE_HOME}/exchangis-engines/engine-plugins/datax mvn clean install ``` 然后会在该路径下找到datax引擎安装包 ``` -{EXCHANGIS_CODE_HOME}/exchangis-plugins/datax/target/out +{EXCHANGIS_CODE_HOME}/exchangis-engines/engine-plugins/target/out ``` @@ -49,7 +49,7 @@ datax 2、放置到linkis安装路径的如下目录 ```shell -cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins +cd {LINKIS_HOME}/linkis-engineconn-plugins ``` （注意，看当前datax引擎对哪些用户有权限，一般都为hadoop用户组和hadoop用户） @@ -60,7 +60,13 @@ cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins cd {LINKIS_INSTALL_HOME}/links/sbin/ ./linkis-daemon.sh restart cg-engineplugin ``` -待服务启动成功，至此，datax安装部署就完成了。 +待服务启动成功，，在linkis数据库中校验datax引擎是否安装完毕 + +```yaml +select * from linkis_cg_engine_conn_plugin_bml_resources where engine_conn_type='datax'; +``` + +至此，datax安装部署就完成了。 engineplugin更详细的介绍可以参看下面的文章。 https://linkis.apache.org/zh-CN/docs/latest/deployment/engine-conn-plugin-installation/ \ No newline at end of file diff --git a/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md b/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md index 20eafb5f7..1439466fe 100644 --- a/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md @@ -37,12 +37,12 @@ Exchangis1.0.0和Linkis 1.1.1支持的主流Sqoop版本1.4.6与1.4.7，更高版 2.在exchangis-plugins模块下，找到sqoop引擎，单独编译sqoop，操作如下 ``` -cd {EXCHANGIS_CODE_HOME}/exchangis-plugins/engine/sqoop +cd {EXCHANGIS_CODE_HOME}/exchangis-engines/engine-plugins/sqoop mvn clean install ``` 然后会在该路径下找到sqoop引擎安装包 ``` -{EXCHANGIS_CODE_HOME}/exchangis-plugins/sqoop/target/out +{EXCHANGIS_CODE_HOME}/exchangis-engines/engine-plugins/sqoop/target/out ``` @@ -59,9 +59,9 @@ sqoop 2.放置到linkis安装路径的如下目录 ```shell -cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins +cd {LINKIS_HOME}/linkis-engineconn-plugins ``` -(注意，看当前sqoop引擎对哪些用户有权限，不一定是root) +（注意，看当前sqoop引擎对哪些用户有权限，不一定是root） #### 2）重启linkis-engineplugin服务使sqoop引擎生效 @@ -70,7 +70,13 @@ cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins cd {LINKIS_INSTALL_HOME}/links/sbin/ ./linkis-daemon.sh restart cg-engineplugin ``` -待服务启动成功，至此，sqoop安装部署就完成了。 +待服务启动成功，在linkis数据库中校验sqoop引擎是否安装完毕 + +```yaml +select * from linkis_cg_engine_conn_plugin_bml_resources; +``` + +至此，sqoop安装部署就完成了。 engineplugin更详细的介绍可以参看下面的文章。 -https://linkis.apache.org/zh-CN/docs/latest/deployment/engine-conn-plugin-installation/ +https://linkis.apache.org/zh-CN/docs/latest/deployment/engine-conn-plugin-installation/ \ No newline at end of file diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobQueryVo.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobQueryVo.java index d0ba5d4f9..3806e8c44 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobQueryVo.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobQueryVo.java @@ -4,8 +4,15 @@ public class ExchangisJobQueryVo extends ExchangisJobPageQuery { + private static final Integer defaultCurrentPage = 1; + + private static final Integer defaultPageSize = 10; + public ExchangisJobQueryVo(){ + } + public ExchangisJobQueryVo(Long projectId, String jobType, String name) { + this(projectId, jobType, name, defaultCurrentPage, defaultPageSize); } public ExchangisJobQueryVo(Long projectId, String jobType, diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java index 57b54fb3b..1025af2e9 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java @@ -236,7 +236,7 @@ public Message updateProject(@Validated({UpdateGroup.class, Default.class}) @Req * @param id project id * @return */ - @RequestMapping( value = "/projects/{id:\\d+}", method = RequestMethod.DELETE) + @DeleteMapping( value = "/projects/{id:\\d+}") public Message deleteProject(HttpServletRequest request, @PathVariable("id") Long id) { if (ExchangisProjectConfiguration.LIMIT_INTERFACE.getValue()) { return Message.error("You have no permission to update (没有编辑权限，无法删除项目)"); @@ -246,12 +246,11 @@ public Message deleteProject(HttpServletRequest request, @PathVariable("id") Lon try { ExchangisProjectInfo projectInfo = projectService.getProjectDetailById(id); if (!ProjectAuthorityUtils.hasProjectAuthority(username, projectInfo, OperationType.PROJECT_ALTER)) { - return Message.error("You have no permission to delete (没有权限删除项目)"); + return Message.error("You have no permission to delete (没有权限删除项目!)"); } // 校验是否有任务 - ExchangisJobQueryVo queryVo = new ExchangisJobQueryVo( - id, null, null, null, null); + ExchangisJobQueryVo queryVo = new ExchangisJobQueryVo(id, null, null); PageResult exchangisJobVoPageResult = jobInfoService.queryJobList(queryVo); if (Objects.nonNull(exchangisJobVoPageResult) && Objects.nonNull(exchangisJobVoPageResult.getList()) && exchangisJobVoPageResult.getList().size() > 0) { From 532705ada7d73a09bd1b51b008b9a02f40308384 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Wed, 26 Jul 2023 22:15:56 +0800 Subject: [PATCH 15/20] Fix the incorrect sum of progress between job and task --- .../job/server/service/impl/DefaultTaskExecuteService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java index b7c914e93..0a35fafcf 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java @@ -132,7 +132,6 @@ public void updateTaskStatus(LaunchedExchangisTask task, TaskStatus status, bool } @Override - @Transactional(rollbackFor = Exception.class) public void updateTaskProgress(LaunchedExchangisTask task, float progress) throws ExchangisOnEventException { task.setLastUpdateTime(Calendar.getInstance().getTime()); this.launchedTaskDao.upgradeLaunchedTaskProgress(task.getTaskId(), progress, task.getLastUpdateTime()); From 31281e0363d1c924fe11a4f023c6dde176b29b39 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Wed, 26 Jul 2023 22:54:37 +0800 Subject: [PATCH 16/20] Adjust the project api --- .../datax/src/main/assembly/distribution.xml | 6 +- .../provider/mapper/impl/ProjectMapper.xml | 2 +- .../restful/ExchangisProjectRestfulApi.java | 13 +- .../ExchangisProjectDssAppConnRestfulApi.java | 200 ++++++++++++------ ...changisProjectJobDssAppConnRestfulApi.java | 109 ++++++++++ ...eExchangisProjectDssAppConnRestfulApi.java | 161 -------------- .../service/impl/ProjectImportServerImpl.java | 3 - 7 files changed, 253 insertions(+), 241 deletions(-) create mode 100644 exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectJobDssAppConnRestfulApi.java delete mode 100644 exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/PreExchangisProjectDssAppConnRestfulApi.java diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml b/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml index 6ac8d1a5c..6c517ef5b 100644 --- a/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml +++ b/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml @@ -294,7 +294,6 @@ - ${basedir}/src/main/resources @@ -302,10 +301,9 @@ 0777 0755 - /dist/v${datax.version}/conf + /dist/${datax.version}/conf unix - ${basedir}/target @@ -317,9 +315,7 @@ 0777 /plugin/${datax.version} - - diff --git a/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectMapper.xml b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectMapper.xml index 70225fb67..dbc90c879 100644 --- a/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectMapper.xml +++ b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectMapper.xml @@ -118,7 +118,7 @@ diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java index 1025af2e9..b6661f1f5 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java @@ -249,19 +249,20 @@ public Message deleteProject(HttpServletRequest request, @PathVariable("id") Lon return Message.error("You have no permission to delete (没有权限删除项目!)"); } + String domain = projectInfo.getDomain(); + if (StringUtils.isNotBlank(domain) && !ExchangisProject.Domain.STANDALONE.name() + .equalsIgnoreCase(domain)){ + return Message.error("Cannot delete the outer project (无法删除来自 " + domain + " 的外部项目)"); + } + // 校验是否有任务 ExchangisJobQueryVo queryVo = new ExchangisJobQueryVo(id, null, null); PageResult exchangisJobVoPageResult = jobInfoService.queryJobList(queryVo); if (Objects.nonNull(exchangisJobVoPageResult) && Objects.nonNull(exchangisJobVoPageResult.getList()) - && exchangisJobVoPageResult.getList().size() > 0) { + && exchangisJobVoPageResult.getList().size() > 0) { return Message.error("Jobs already exist under this project and the project cannot be deleted (该项目下已存在子任务，无法删除)"); } - String domain = projectInfo.getDomain(); - if (StringUtils.isNotBlank(domain) && !ExchangisProject.Domain.STANDALONE.name() - .equalsIgnoreCase(domain)){ - return Message.error("Cannot delete the outer project (无法删除来自 " + domain + " 的外部项目)"); - } projectService.deleteProject(id); AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, id.toString(), "Project", OperateTypeEnum.DELETE, request); return ExchangisProjectRestfulUtils.dealOk("删除项目成功"); diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java index 6221c71e1..2e8457219 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java @@ -4,106 +4,176 @@ import com.webank.wedatasphere.exchangis.common.UserUtils; import com.webank.wedatasphere.exchangis.common.enums.OperateTypeEnum; import com.webank.wedatasphere.exchangis.common.enums.TargetTypeEnum; -import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.project.provider.mapper.ProjectMapper; -import com.webank.wedatasphere.exchangis.project.server.service.ProjectCopyService; -import com.webank.wedatasphere.exchangis.project.server.service.impl.ProjectExportServiceImpl; -import com.webank.wedatasphere.exchangis.project.server.service.impl.ProjectImportServerImpl; +import com.webank.wedatasphere.exchangis.common.pager.PageResult; +import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; +import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; +import com.webank.wedatasphere.exchangis.project.entity.domain.OperationType; +import com.webank.wedatasphere.exchangis.project.entity.vo.ExchangisProjectAppVo; +import com.webank.wedatasphere.exchangis.project.entity.vo.ExchangisProjectInfo; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; +import com.webank.wedatasphere.exchangis.project.server.utils.ExchangisProjectRestfulUtils; +import com.webank.wedatasphere.exchangis.project.server.utils.ProjectAuthorityUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.math3.util.Pair; +import org.apache.linkis.common.utils.JsonUtils; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.BindingResult; +import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.core.Context; -import java.rmi.ServerException; -import java.util.Map; +import javax.validation.groups.Default; +import java.util.Objects; /** - * Define to support the app conn, in order to distinguish from the inner api + * Restful class for dss project */ @RestController -@RequestMapping(value = "/dss/exchangis/main/appJob", produces = {"application/json;charset=utf-8"}) +@RequestMapping(value = "/dss/exchangis/main/appProject", produces = {"application/json;charset=utf-8"}) public class ExchangisProjectDssAppConnRestfulApi { - private static final Logger LOG = LoggerFactory.getLogger(ExchangisProjectDssAppConnRestfulApi.class); + /** + * Project service + */ @Resource - private ProjectImportServerImpl projectImportServer; + private ProjectService projectService; + /** + * JobInfo service + */ @Resource - private ProjectExportServiceImpl projectExportService; + private JobInfoService jobInfoService; - @Resource - private ProjectCopyService projectCopyService; + @RequestMapping(value = "", method = RequestMethod.POST) + public Message createProject(@Validated @RequestBody ExchangisProjectAppVo project, + BindingResult result, HttpServletRequest request){ + ExchangisProjectInfo projectVo = new ExchangisProjectInfo(project); + if (result.hasErrors()){ + return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); + } - @Autowired - private ProjectMapper projectMapper; + String oringinUser = SecurityFilter.getLoginUsername(request); + String username = UserUtils.getLoginUser(request); + if (StringUtils.isBlank(projectVo.getViewUsers()) || !StringUtils.contains(projectVo.getViewUsers(), username)) { + projectVo.setViewUsers(username + projectVo.getViewUsers()); + } + if (StringUtils.isBlank(projectVo.getEditUsers()) || !StringUtils.contains(projectVo.getEditUsers(), username)) { + projectVo.setEditUsers(username + projectVo.getEditUsers()); + } + if (StringUtils.isBlank(projectVo.getExecUsers()) || !StringUtils.contains(projectVo.getExecUsers(), username)) { + projectVo.setExecUsers(username + projectVo.getExecUsers()); - @RequestMapping(value = "/import", method = RequestMethod.POST) - public Message importJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { + } - Message response = null; - String userName = UserUtils.getLoginUser(request); - String oringinUser = SecurityFilter.getLoginUsername(request); try { - LOG.info("param: {}", params); - /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { - return Message.error("You have no permission to import (没有导入权限)"); - }*/ - response = projectImportServer.importProject(request, params); - LOG.info("import job success"); - } catch (ExchangisJobServerException e) { - String message = "Fail import job [ id: " + params + "] (导入任务失败)"; - LOG.error(message, e); - response = Message.error(message); + LOG.info("CreateProject from DSS AppConn, vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); + if (projectService.existsProject(null, projectVo.getName())){ + return Message.error("Have the same name project (存在同名工程)"); + } + long projectIdd = projectService.createProject(projectVo, username); + String projectId = String.valueOf(projectIdd); + AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, String.valueOf(projectId), "Project name is: " + projectVo.getName(), OperateTypeEnum.CREATE, request); + return ExchangisProjectRestfulUtils.dealOk("创建工程成功", + new Pair<>("projectName", projectVo.getName()), + new Pair<>("projectId", projectId)); + } catch (Exception t) { + LOG.error("Failed to create project for user {} from DSS", username, t); + return Message.error("Fail to create project from DSS(创建工程失败)"); } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.IMPORT, request); - return response; - } - @RequestMapping(value = "/export", method = RequestMethod.POST) - public Message exportJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { - String userName = UserUtils.getLoginUser(request); - LOG.info("export function params: {}", params); + /** + * Update project + * @param request request + * @param projectVo project vo + * @return + */ + @RequestMapping( value = "/{id:\\d+}", method = RequestMethod.PUT) + public Message updateProject(@PathVariable("id") Long id, @Validated({UpdateGroup.class, Default.class}) @RequestBody ExchangisProjectInfo projectVo + , BindingResult result, HttpServletRequest request) { + if (result.hasErrors()){ + return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); + } + String username = UserUtils.getLoginUser(request); String oringinUser = SecurityFilter.getLoginUsername(request); - Message response = null; try { - /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { - return Message.error("You have no permission to export (没有导出权限)"); - }*/ - response = projectExportService.exportProject(params, userName, request); - LOG.info("export job success"); - } catch (Exception e) { - String message = "Fail Export job [ id: " + params + "] (导出任务失败)"; - LOG.error(message, e); - response = Message.error(message); + ExchangisProjectInfo projectStored = projectService.getProjectDetailById(Long.valueOf(projectVo.getId())); + if (!ProjectAuthorityUtils.hasProjectAuthority(username, projectStored, OperationType.PROJECT_ALTER)) { + return Message.error("You have no permission to update (没有项目的更新权限)"); + } + + LOG.info("UpdateProject vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); + projectService.updateProject(projectVo, username); + AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, id.toString(), "Project name is: " + projectVo.getName(), OperateTypeEnum.UPDATE, request); + return ExchangisProjectRestfulUtils.dealOk("更新工程成功", + new Pair<>("projectName", projectVo.getName()), + new Pair<>("projectId", projectVo.getId())); + } catch (Exception t) { + LOG.error("Failed to update project for user {}", username, t); + return Message.error("Fail to update project (更新工程失败)"); } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.EXPORT, request); - return response; } - @RequestMapping(value = "/copy", method = RequestMethod.POST) - public Message copy(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException { - String userName = UserUtils.getLoginUser(request); + /** + * Delete project + * @param request http request + * @param name project name + * @return + */ + @RequestMapping( value = "/{name}", method = RequestMethod.POST) + public Message deleteProject(HttpServletRequest request, @PathVariable("name") String name) { + String username = UserUtils.getLoginUser(request); String oringinUser = SecurityFilter.getLoginUsername(request); + try { + ExchangisProjectInfo projectInfo = projectService.selectByName(name); + if (!ProjectAuthorityUtils.hasProjectAuthority(username, projectInfo, OperationType.PROJECT_ALTER)) { + return Message.error("You have no permission to delete (删除项目失败)"); + } + + // 校验是否有任务 + ExchangisJobQueryVo queryVo = new ExchangisJobQueryVo(Long.parseLong(projectInfo.getId()), null, null); + PageResult exchangisJobVoPageResult = jobInfoService.queryJobList(queryVo); + if (Objects.nonNull(exchangisJobVoPageResult) && Objects.nonNull(exchangisJobVoPageResult.getList()) + && exchangisJobVoPageResult.getList().size() > 0) { + return Message.error("Jobs already exist under this project and the project cannot be deleted (该项目下已存在子任务，无法删除)"); + } + + projectService.deleteProjectByName(name); + AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, "", "Project name is: " + name, OperateTypeEnum.DELETE, request); + return ExchangisProjectRestfulUtils.dealOk("删除工程成功"); + } catch (Exception t) { + LOG.error("Failed to delete project for user {}", username, t); + return Message.error("Failed to delete project (删除工程失败)"); + } - LOG.info("copy function params: {}", params); - Message response = null; + } + + + /** + * check project name + * @param request http request + * @param name project name + * @return + */ + @RequestMapping( value = "/check/{name}", method = RequestMethod.POST) + public Message getProjectByName(HttpServletRequest request, @PathVariable("name") String name) { + String username = UserUtils.getLoginUser(request); try { - response = projectCopyService.copy(params, userName, request); - LOG.info("copy node success"); - } catch (Exception e) { - String message = "Fail Copy project [ id: " + params + "] (导出任务失败)"; - LOG.error(message, e); - response = Message.error(message); + ExchangisProjectInfo projectInfo = projectService.selectByName(name); + return ExchangisProjectRestfulUtils.dealOk("根据名字获取工程成功", + new Pair<>("projectInfo",projectInfo)); + } catch (Exception t) { + LOG.error("Failed to delete project for user {}", username, t); + return Message.error("Failed to delete project (根据名字获取工程失败)"); } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Copy parameter is: " + params.toString(), OperateTypeEnum.COPY, request); - return response; } + } diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectJobDssAppConnRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectJobDssAppConnRestfulApi.java new file mode 100644 index 000000000..fc23ec0e0 --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectJobDssAppConnRestfulApi.java @@ -0,0 +1,109 @@ +package com.webank.wedatasphere.exchangis.project.server.restful.external; + +import com.webank.wedatasphere.exchangis.common.AuditLogUtils; +import com.webank.wedatasphere.exchangis.common.UserUtils; +import com.webank.wedatasphere.exchangis.common.enums.OperateTypeEnum; +import com.webank.wedatasphere.exchangis.common.enums.TargetTypeEnum; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.project.provider.mapper.ProjectMapper; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectCopyService; +import com.webank.wedatasphere.exchangis.project.server.service.impl.ProjectExportServiceImpl; +import com.webank.wedatasphere.exchangis.project.server.service.impl.ProjectImportServerImpl; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.Context; +import java.rmi.ServerException; +import java.util.Map; + +/** + * Define to support the app conn, in order to distinguish from the inner api + */ +@RestController +@RequestMapping(value = "/dss/exchangis/main/appJob", produces = {"application/json;charset=utf-8"}) +public class ExchangisProjectJobDssAppConnRestfulApi { + + private static final Logger LOG = LoggerFactory.getLogger(ExchangisProjectJobDssAppConnRestfulApi.class); + + @Resource + private ProjectImportServerImpl projectImportServer; + + @Resource + private ProjectExportServiceImpl projectExportService; + + @Resource + private ProjectCopyService projectCopyService; + + @Autowired + private ProjectMapper projectMapper; + + @RequestMapping(value = "/import", method = RequestMethod.POST) + public Message importJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { + + Message response = null; + String userName = UserUtils.getLoginUser(request); + String oringinUser = SecurityFilter.getLoginUsername(request); + try { + LOG.info("param: {}", params); + /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { + return Message.error("You have no permission to import (没有导入权限)"); + }*/ + response = projectImportServer.importProject(request, params); + LOG.info("import job success"); + } catch (ExchangisJobServerException e) { + String message = "Fail import job [ id: " + params + "] (导入任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.IMPORT, request); + return response; + + } + + @RequestMapping(value = "/export", method = RequestMethod.POST) + public Message exportJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { + String userName = UserUtils.getLoginUser(request); + + LOG.info("export function params: {}", params); + String oringinUser = SecurityFilter.getLoginUsername(request); + Message response = null; + try { + /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { + return Message.error("You have no permission to export (没有导出权限)"); + }*/ + response = projectExportService.exportProject(params, userName, request); + LOG.info("export job success"); + } catch (Exception e) { + String message = "Fail Export job [ id: " + params + "] (导出任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.EXPORT, request); + return response; + } + + @RequestMapping(value = "/copy", method = RequestMethod.POST) + public Message copy(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException { + String userName = UserUtils.getLoginUser(request); + String oringinUser = SecurityFilter.getLoginUsername(request); + + LOG.info("copy function params: {}", params); + Message response = null; + try { + response = projectCopyService.copy(params, userName, request); + LOG.info("copy node success"); + } catch (Exception e) { + String message = "Fail Copy project [ id: " + params + "] (导出任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Copy parameter is: " + params.toString(), OperateTypeEnum.COPY, request); + return response; + } +} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/PreExchangisProjectDssAppConnRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/PreExchangisProjectDssAppConnRestfulApi.java deleted file mode 100644 index 0212d6b3d..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/PreExchangisProjectDssAppConnRestfulApi.java +++ /dev/null @@ -1,161 +0,0 @@ -//package com.webank.wedatasphere.exchangis.project.server.restful.external; -// -//import com.webank.wedatasphere.exchangis.common.AuditLogUtils; -//import com.webank.wedatasphere.exchangis.common.UserUtils; -//import com.webank.wedatasphere.exchangis.common.enums.OperateTypeEnum; -//import com.webank.wedatasphere.exchangis.common.enums.TargetTypeEnum; -//import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; -//import com.webank.wedatasphere.exchangis.project.server.domain.OperationType; -//import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; -//import com.webank.wedatasphere.exchangis.project.server.utils.ExchangisProjectRestfulUtils; -//import com.webank.wedatasphere.exchangis.project.server.utils.ProjectAuthorityUtils; -//import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectAppVo; -//import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; -//import org.apache.commons.lang3.StringUtils; -//import org.apache.commons.math3.util.Pair; -//import org.apache.linkis.common.utils.JsonUtils; -//import org.apache.linkis.server.Message; -//import org.apache.linkis.server.security.SecurityFilter; -//import org.slf4j.Logger; -//import org.slf4j.LoggerFactory; -//import org.springframework.validation.BindingResult; -//import org.springframework.validation.annotation.Validated; -//import org.springframework.web.bind.annotation.*; -// -//import javax.annotation.Resource; -//import javax.servlet.http.HttpServletRequest; -//import javax.validation.groups.Default; -// -///** -// * Restful class for dss project -// */ -//@Deprecated -//@RestController -//@RequestMapping(value = "/dss/exchangis/main/appProject", produces = {"application/json;charset=utf-8"}) -//public class PreExchangisProjectDssAppConnRestfulApi { -// private static final Logger LOG = LoggerFactory.getLogger(PreExchangisProjectDssAppConnRestfulApi.class); -// -// /** -// * Project service -// */ -// @Resource -// private ProjectService projectService; -// -// @RequestMapping(value = "", method = RequestMethod.POST) -// public Message createProject(@Validated @RequestBody ExchangisProjectAppVo project, -// BindingResult result, HttpServletRequest request){ -// ExchangisProjectInfo projectVo = new ExchangisProjectInfo(project); -// if (result.hasErrors()){ -// return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); -// } -// -// String oringinUser = SecurityFilter.getLoginUsername(request); -// String username = UserUtils.getLoginUser(request); -// if (StringUtils.isBlank(projectVo.getViewUsers()) || !StringUtils.contains(projectVo.getViewUsers(), username)) { -// projectVo.setViewUsers(username + projectVo.getViewUsers()); -// } -// if (StringUtils.isBlank(projectVo.getEditUsers()) || !StringUtils.contains(projectVo.getEditUsers(), username)) { -// projectVo.setEditUsers(username + projectVo.getEditUsers()); -// } -// if (StringUtils.isBlank(projectVo.getExecUsers()) || !StringUtils.contains(projectVo.getExecUsers(), username)) { -// projectVo.setExecUsers(username + projectVo.getExecUsers()); -// -// } -// -// try { -// LOG.info("CreateProject from DSS AppConn, vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); -// if (projectService.existsProject(null, projectVo.getName())){ -// return Message.error("Have the same name project (存在同名工程)"); -// } -// long projectIdd = projectService.createProject(projectVo, username); -// String projectId = String.valueOf(projectIdd); -// AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, String.valueOf(projectId), "Project name is: " + projectVo.getName(), OperateTypeEnum.CREATE, request); -// return ExchangisProjectRestfulUtils.dealOk("创建工程成功", -// new Pair<>("projectName", projectVo.getName()), -// new Pair<>("projectId", projectId)); -// } catch (Exception t) { -// LOG.error("Failed to create project for user {} from DSS", username, t); -// return Message.error("Fail to create project from DSS(创建工程失败)"); -// } -// } -// -// -// /** -// * Update project -// * @param request request -// * @param projectVo project vo -// * @return -// */ -// @RequestMapping( value = "/{id:\\d+}", method = RequestMethod.PUT) -// public Message updateProject(@PathVariable("id") Long id, @Validated({UpdateGroup.class, Default.class}) @RequestBody ExchangisProjectInfo projectVo -// , BindingResult result, HttpServletRequest request) { -// if (result.hasErrors()){ -// return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); -// } -// String username = UserUtils.getLoginUser(request); -// String oringinUser = SecurityFilter.getLoginUsername(request); -// try { -// ExchangisProjectInfo projectStored = projectService.getProjectDetailById(Long.valueOf(projectVo.getId())); -// if (!ProjectAuthorityUtils.hasProjectAuthority(username, projectStored, OperationType.PROJECT_ALTER)) { -// return Message.error("You have no permission to update (没有项目的更新权限)"); -// } -// -// LOG.info("UpdateProject vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); -// projectService.updateProject(projectVo, username); -// AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, id.toString(), "Project name is: " + projectVo.getName(), OperateTypeEnum.UPDATE, request); -// return ExchangisProjectRestfulUtils.dealOk("更新工程成功", -// new Pair<>("projectName", projectVo.getName()), -// new Pair<>("projectId", projectVo.getId())); -// } catch (Exception t) { -// LOG.error("Failed to update project for user {}", username, t); -// return Message.error("Fail to update project (更新工程失败)"); -// } -// } -// -// /** -// * Delete project -// * @param request http request -// * @param name project name -// * @return -// */ -// @RequestMapping( value = "/{name}", method = RequestMethod.POST) -// public Message deleteProject(HttpServletRequest request, @PathVariable("name") String name) { -// String username = UserUtils.getLoginUser(request); -// String oringinUser = SecurityFilter.getLoginUsername(request); -// try { -// ExchangisProjectInfo projectInfo = projectService.selectByName(name); -// if (!ProjectAuthorityUtils.hasProjectAuthority(username, projectInfo, OperationType.PROJECT_ALTER)) { -// return Message.error("You have no permission to delete (删除项目失败)"); -// } -// -// projectService.deleteProjectByName(name); -// AuditLogUtils.printLog(oringinUser, username, TargetTypeEnum.PROJECT, "", "Project name is: " + name, OperateTypeEnum.DELETE, request); -// return ExchangisProjectRestfulUtils.dealOk("删除工程成功"); -// } catch (Exception t) { -// LOG.error("Failed to delete project for user {}", username, t); -// return Message.error("Failed to delete project (删除工程失败)"); -// } -// -// } -// -// -// /** -// * check project name -// * @param request http request -// * @param name project name -// * @return -// */ -// @RequestMapping( value = "/check/{name}", method = RequestMethod.POST) -// public Message getProjectByName(HttpServletRequest request, @PathVariable("name") String name) { -// String username = UserUtils.getLoginUser(request); -// try { -// ExchangisProjectInfo projectInfo = projectService.selectByName(name); -// return ExchangisProjectRestfulUtils.dealOk("根据名字获取工程成功", -// new Pair<>("projectInfo",projectInfo)); -// } catch (Exception t) { -// LOG.error("Failed to delete project for user {}", username, t); -// return Message.error("Failed to delete project (根据名字获取工程失败)"); -// } -// } -// -//} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectImportServerImpl.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectImportServerImpl.java index 6ca3ab4cb..acae2fa98 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectImportServerImpl.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectImportServerImpl.java @@ -150,9 +150,6 @@ else if (projects.size() == 1) { else { throw new ExchangisJobServerException(31101, "Already exits duplicated project name(存在重复项目名称) projectName is:" + "[" + exportedProject.getName() + "]"); } - - - } public void addSqoopTask (List sqoops, List projects, String versionSuffix, IdCatalog idCatalog, Long projectId, String importType) throws ExchangisJobServerException { From cad221c6c0792f7cfbb5d91eade7c417b1f075ae Mon Sep 17 00:00:00 2001 From: jefftlin Date: Thu, 3 Aug 2023 21:19:37 +0800 Subject: [PATCH 17/20] Upgrade version of hadoop to 3.3.4 and hive to 3.1.3 --- docs/en_US/ch1/exchangis_deploy_en.md | 4 +-- docs/zh_CN/ch1/exchangis_datax_deploy_cn.md | 10 +++---- docs/zh_CN/ch1/exchangis_deploy_cn.md | 4 +-- docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md | 2 +- .../engineconn-plugins/datax/pom.xml | 29 +++++++++++++++++-- .../datax/DataxEngineConnPlugin.scala | 3 +- .../engineconn-plugins/sqoop/pom.xml | 8 +++-- .../engines/datax/datax-core/pom.xml | 1 + .../engines/datax/datax-hdfsreader/pom.xml | 11 ++++--- .../hdfsreader/hfile/HBASEV1HFileParser.java | 7 ++--- .../plugin/reader/hdfsreader/HFileParser.java | 8 ++--- .../engines/datax/datax-hdfswriter/pom.xml | 5 ++-- exchangis-engines/engines/datax/pom.xml | 4 +-- .../engine/resource/EngineResourceLoader.java | 4 +-- .../mapper/impl/ProjectUserMapper.xml | 2 +- pom.xml | 2 +- 16 files changed, 66 insertions(+), 38 deletions(-) diff --git a/docs/en_US/ch1/exchangis_deploy_en.md b/docs/en_US/ch1/exchangis_deploy_en.md index c80d02484..c5a1b2564 100644 --- a/docs/en_US/ch1/exchangis_deploy_en.md +++ b/docs/en_US/ch1/exchangis_deploy_en.md @@ -15,7 +15,7 @@ Exchangis installation is mainly divided into the following four steps ： |------------------------------------------------------------------------------| ------ | --------------- | | MySQL (5.5+) | yes | [How to install mysql](https://www.runoob.com/mysql/mysql-install.html) | | JDK (1.8.0_141) | yes | [How to install JDK](https://www.runoob.com/java/java-environment-setup.html) | -| Hadoop(2.7.2，Other versions of Hadoop need to compile Linkis by themselves.) | yes | [Hadoop stand-alone deployment](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) ；[Hadoop distributed deployment](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | +| Hadoop(3.3.4，Other versions of Hadoop need to compile Linkis by themselves.) | yes | [Hadoop stand-alone deployment](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) ；[Hadoop distributed deployment](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | | Hive(2.3.3，Other versions of Hive need to compile Linkis by themselves.) | yes | [Hive quick installation](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | | SQOOP (1.4.6) | yes | [How to install Sqoop](https://sqoop.apache.org/docs/1.4.6/SqoopUserGuide.html) | | DSS1.1.2 | yes | [How to install DSS](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) | @@ -205,7 +205,7 @@ Get the compiled dist.zip front-end package from the `web/` path. The acquired front-end package can be placed anywhere on the server. Here, it is recommended that you keep the same directory as the back-end installation address, place it in the same directory and unzip it. -#### 2.7.2 Front-end installation deployment +#### 3.3.4 Front-end installation deployment 1. Decompress front-end installation package diff --git a/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md b/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md index df854bd42..81c1a821d 100644 --- a/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md @@ -27,12 +27,12 @@ Exchangis1.1.2和Linkis 1.4.0支持的主流DataX版本1.4.6与1.4.7，更高版 2、在exchangis-plugins模块下，找到datax引擎，单独编译datax，操作如下 ``` -cd {EXCHANGIS_CODE_HOME}/exchangis-engines/engine-plugins/datax +cd {EXCHANGIS_CODE_HOME}/exchangis-plugins/engine/datax mvn clean install ``` 然后会在该路径下找到datax引擎安装包 ``` -{EXCHANGIS_CODE_HOME}/exchangis-engines/engine-plugins/target/out +{EXCHANGIS_CODE_HOME}/exchangis-plugins/datax/target/out ``` @@ -49,7 +49,7 @@ datax 2、放置到linkis安装路径的如下目录 ```shell -cd {LINKIS_HOME}/linkis-engineconn-plugins +cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins ``` （注意，看当前datax引擎对哪些用户有权限，一般都为hadoop用户组和hadoop用户） @@ -60,9 +60,9 @@ cd {LINKIS_HOME}/linkis-engineconn-plugins cd {LINKIS_INSTALL_HOME}/links/sbin/ ./linkis-daemon.sh restart cg-engineplugin ``` -待服务启动成功，，在linkis数据库中校验datax引擎是否安装完毕 +待服务启动成功，在linkis数据库中校验datax引擎是否安装完毕 -```yaml +```shell select * from linkis_cg_engine_conn_plugin_bml_resources where engine_conn_type='datax'; ``` diff --git a/docs/zh_CN/ch1/exchangis_deploy_cn.md b/docs/zh_CN/ch1/exchangis_deploy_cn.md index 2ec1d41e1..6d2937210 100644 --- a/docs/zh_CN/ch1/exchangis_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_deploy_cn.md @@ -16,7 +16,7 @@ Exchangis 的安装，主要分为以下四步： |---------------------------------------| ------ | --------------- | | MySQL (5.5+) | 必装 | [如何安装mysql](https://www.runoob.com/mysql/mysql-install.html) | | JDK (1.8.0_141) | 必装 | [如何安装JDK](https://www.runoob.com/java/java-environment-setup.html) | -| Hadoop(2.7.2，Hadoop 其他版本需自行编译 Linkis) | 必装 | [Hadoop单机部署](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) ；[Hadoop分布式部署](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | +| Hadoop(3.3.4，Hadoop 其他版本需自行编译 Linkis) | 必装 | [Hadoop单机部署](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) ；[Hadoop分布式部署](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | | Hive(2.3.3，Hive 其他版本需自行编译 Linkis) | 必装 | [Hive快速安装](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | | SQOOP (1.4.6) | 必装 | [如何安装Sqoop](https://sqoop.apache.org/docs/1.4.6/SqoopUserGuide.html) | | DSS1.1.2 | 必装 | [如何安装DSS](https://github.com/WeBankFinTech/DataSphereStudio-Doc/tree/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2) | @@ -223,7 +223,7 @@ Exchangis 已默认提供了编译好的前端安装包，可直接下载使用 获取到的前端包，您可以放在服务器上的任意位置，这里建议您与后端安装地址目录保持一致，在同一目录下放置并解压。 -#### 2.7.2 前端安装部署 +#### 3.3.4 前端安装部署 1. 解压前端安装包 diff --git a/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md b/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md index 1439466fe..817854020 100644 --- a/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md @@ -73,7 +73,7 @@ cd {LINKIS_INSTALL_HOME}/links/sbin/ 待服务启动成功，在linkis数据库中校验sqoop引擎是否安装完毕 ```yaml -select * from linkis_cg_engine_conn_plugin_bml_resources; +select * from linkis_cg_engine_conn_plugin_bml_resources where engine_conn_type='sqoop'; ``` 至此，sqoop安装部署就完成了。 diff --git a/exchangis-engines/engineconn-plugins/datax/pom.xml b/exchangis-engines/engineconn-plugins/datax/pom.xml index 392671b64..209e59f0a 100644 --- a/exchangis-engines/engineconn-plugins/datax/pom.xml +++ b/exchangis-engines/engineconn-plugins/datax/pom.xml @@ -16,7 +16,7 @@ 1.4.0-SNAPSHOT 3.0.0-Plus-2 3.0.0 - 2.7.2 + 3.3.4 1.0.15 2.1.9 0.3 @@ -90,7 +90,6 @@ org.apache.linkis linkis-common ${linkis.version} - provided commons-cli @@ -102,6 +101,21 @@ commons-configuration ${commons-configuration-version} + + org.apache.linkis + linkis-computation-engineconn + 1.4.0-SNAPSHOT + + + org.apache.linkis + linkis-udf-common + 1.4.0-SNAPSHOT + + + org.apache.linkis + linkis-udf-client + 1.4.0-SNAPSHOT + @@ -176,4 +190,15 @@ + + + + apache.snapshots + Apache Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + true + + + \ No newline at end of file diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala index fb3ac3f68..996e4da90 100644 --- a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala +++ b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala @@ -26,7 +26,6 @@ import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuil import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceFactory import org.apache.linkis.manager.label.entity.Label -import java.util.Map import java.util.List import java.util.ArrayList @@ -38,7 +37,7 @@ class DataxEngineConnPlugin extends EngineConnPlugin { private var engineConnFactory: EngineConnFactory = _ private val engineConnFactoryLocker = new Array[Byte](0) - override def init(params: Map[String, AnyRef]): Unit = {} + override def init(params: java.util.Map[_root_.scala.Predef.String, scala.AnyRef]): Unit = {} override def getEngineResourceFactory: EngineResourceFactory = { if (null == engineResourceFactory) engineResourceFactoryLocker.synchronized { diff --git a/exchangis-engines/engineconn-plugins/sqoop/pom.xml b/exchangis-engines/engineconn-plugins/sqoop/pom.xml index 8e8559c48..49aa7117f 100644 --- a/exchangis-engines/engineconn-plugins/sqoop/pom.xml +++ b/exchangis-engines/engineconn-plugins/sqoop/pom.xml @@ -30,8 +30,8 @@ linkis-engineplugin-sqoop 1.4.6 - 3.1.2 - 2.7.2 + 3.1.3 + 3.3.4 1.4.0-SNAPSHOT @@ -180,6 +180,10 @@ hadoop-hdfs org.apache.hadoop + + org.apache.hadoop + hadoop-hdfs + diff --git a/exchangis-engines/engines/datax/datax-core/pom.xml b/exchangis-engines/engines/datax/datax-core/pom.xml index 491136a65..48be143d3 100644 --- a/exchangis-engines/engines/datax/datax-core/pom.xml +++ b/exchangis-engines/engines/datax/datax-core/pom.xml @@ -79,6 +79,7 @@ ch.qos.logback logback-classic + provided org.slf4j diff --git a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml index 737641292..81789b25b 100644 --- a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml @@ -13,13 +13,12 @@ jar 3.0.0-Plus-2 - 1.2.1 - 2.7.2 - 1.2.2 + 3.1.3 + 3.3.4 + 2.5.5 2.9.1 - com.webank.wedatasphere.exchangis datax-core @@ -108,6 +107,10 @@ hbase-server ${hbase.version} + + org.apache.hadoop + hadoop-hdfs + diff --git a/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/hfile/HBASEV1HFileParser.java b/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/hfile/HBASEV1HFileParser.java index 0fe8d4a69..b46189e7c 100644 --- a/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/hfile/HBASEV1HFileParser.java +++ b/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/hfile/HBASEV1HFileParser.java @@ -47,19 +47,18 @@ public void parse(String inputPath, Configuration parseConf, Action action) { org.apache.hadoop.conf.Configuration configuration = fileSystem.getConf(); LOG.info("Start to parse HFile: [" + inputPath + "] in HBASEV1HFileParser"); try (HFile.Reader reader = HFile.createReader(fileSystem, new Path(inputPath), - new CacheConfig(configuration), configuration)) { - HFileScanner scanner = reader.getScanner(true, true); + new CacheConfig(configuration), false, configuration)) { + HFileScanner scanner = reader.getScanner(configuration, true, true); if(null == parseConf){ parseConf = Configuration.from("{}"); for(String parseColumn : PARSE_COLUMNS){ parseConf.set(parseColumn, true); } } - reader.loadFileInfo(); if(scanner.seekTo()) { do { //Cell entity - Cell cell = scanner.getKeyValue(); + Cell cell = scanner.getCell(); List sourceList = new ArrayList<>(); parseConf.getKeys().forEach(configKey -> { switch(configKey){ diff --git a/exchangis-engines/engines/datax/datax-hdfsreader/src/test/java/com/alibaba/datax/plugin/reader/hdfsreader/HFileParser.java b/exchangis-engines/engines/datax/datax-hdfsreader/src/test/java/com/alibaba/datax/plugin/reader/hdfsreader/HFileParser.java index d40a9644c..dae334eab 100644 --- a/exchangis-engines/engines/datax/datax-hdfsreader/src/test/java/com/alibaba/datax/plugin/reader/hdfsreader/HFileParser.java +++ b/exchangis-engines/engines/datax/datax-hdfsreader/src/test/java/com/alibaba/datax/plugin/reader/hdfsreader/HFileParser.java @@ -4,7 +4,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; @@ -15,11 +14,10 @@ public class HFileParser { public void parse(String pathString, Configuration hadoopConf) throws IOException { FileSystem fs = new Path(pathString).getFileSystem(hadoopConf); - HFile.Reader reader = HFile.createReader(fs, new Path(pathString), new CacheConfig(hadoopConf), hadoopConf); - HFileScanner scanner = reader.getScanner(true, true); - reader.loadFileInfo(); + HFile.Reader reader = HFile.createReader(fs, new Path(pathString), new CacheConfig(hadoopConf), false, hadoopConf); + HFileScanner scanner = reader.getScanner(hadoopConf, true, true); scanner.seekTo(); - Cell cell = scanner.getKeyValue(); + Cell cell = scanner.getCell(); scanner.next(); } } diff --git a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml index a8f25054b..11c814aec 100644 --- a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml @@ -14,8 +14,8 @@ jar 3.0.0-Plus-2 - 1.2.1 - 2.7.2 + 3.1.3 + 3.3.4 2.9.1 @@ -46,7 +46,6 @@ logback-classic provided - org.apache.hadoop hadoop-hdfs diff --git a/exchangis-engines/engines/datax/pom.xml b/exchangis-engines/engines/datax/pom.xml index 3cef763af..15d7f5879 100644 --- a/exchangis-engines/engines/datax/pom.xml +++ b/exchangis-engines/engines/datax/pom.xml @@ -25,8 +25,8 @@ 1.10 1.2 1.9.4 - 2.7.2 - 1.2.1 + 3.3.4 + 3.1.3 6.7.1 1.11 3.0.0-Plus-2 diff --git a/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/EngineResourceLoader.java b/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/EngineResourceLoader.java index ca948ae98..dd5a74241 100644 --- a/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/EngineResourceLoader.java +++ b/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/EngineResourceLoader.java @@ -17,14 +17,14 @@ public interface EngineResourceLoader { String engineType(); /** * Accept uri - * @param uri uri + * @param baseUri uri * @return boolean */ boolean accept(URI baseUri, String path); /** * Load resources from uri - * @param uri uri + * @param baseUri uri * @return resource array */ T[] loadResource(URI baseUri, String path) throws ExchangisEngineResLoadException; diff --git a/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectUserMapper.xml b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectUserMapper.xml index 9a8e2d257..f4b3709a6 100644 --- a/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectUserMapper.xml +++ b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectUserMapper.xml @@ -22,7 +22,7 @@ delete from - where project_id = #{projectId}; + where project_id = #{projectId}