diff --git a/assembly-package/config/application-eureka.yml b/assembly-package/config/application-eureka.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/assembly-package/config/exchangis-server.properties b/assembly-package/config/exchangis-server.properties deleted file mode 100644 index f2c12f8bc..000000000 --- a/assembly-package/config/exchangis-server.properties +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -wds.linkis.server.mybatis.datasource.url=jdbc:mysql://{IP}:{PORT}/{database}?useSSL=false&characterEncoding=UTF-8&allowMultiQueries=true&useAffectedRows=true - -wds.linkis.server.mybatis.datasource.username={username} - -wds.linkis.server.mybatis.datasource.password={password} - -wds.linkis.log.clear=true - -wds.linkis.server.version=v1 - -# datasource client -wds.exchangis.datasource.client.serverurl=http://{LINKIS_IP}:{LINKIS_PORT}/ -wds.exchangis.datasource.client.authtoken.key=EXCHANGIS-AUTH -wds.exchangis.datasource.client.authtoken.value=EXCHANGIS-AUTH -wds.exchangis.datasource.client.dws.version=v1 - -# launcher client -wds.exchangis.client.linkis.server-url=http://{LINKIS_IP}:{LINKIS_PORT}/ -wds.exchangis.client.linkis.token.value=EXCHANGIS-AUTH -wds.exchangis.datasource.extension.dir=exchangis-extds/ - -##restful -wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.exchangis.datasource.server.restful.api,\ - com.webank.wedatasphere.exchangis.project.server.restful,\ - com.webank.wedatasphere.exchangis.job.server.restful -wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/workflow/dao/impl/*.xml,\ -classpath*:com/webank/wedatasphere/exchangis/job/server/mapper/impl/*.xml,\ -classpath*:com/webank/wedatasphere/exchangis/project/server/mapper/impl/*.xml,\ -classpath*:com/webank/wedatasphere/exchangis/engine/server/mapper/*.xml - -wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.exchangis.dao,\ - com.webank.wedatasphere.exchangis.project.server.mapper,\ - com.webank.wedatasphere.linkis.configuration.dao,\ - com.webank.wedatasphere.dss.framework.appconn.dao,\ - com.webank.wedatasphere.dss.workflow.dao,\ - com.webank.wedatasphere.linkis.metadata.dao,\ - com.webank.wedatasphere.exchangis.job.server.mapper,\ - com.webank.wedatasphere.exchangis.job.server.dao,\ - com.webank.wedatasphere.exchangis.engine.dao - -wds.exchangis.job.task.scheduler.load-balancer.flexible.segments.min-occupy=0.25 -wds.exchangis.job.task.scheduler.load-balancer.flexible.segments.max-occupy=0.5 -#wds.exchangis.job.scheduler.group.max.running-jobs=4 - -wds.linkis-session.ticket.key=bdp-user-ticket-id -wds.exchangis.limit.interface.value=false - -wds.exchangis.publicKeyStr= -wds.exchangis.privateKeyStr= diff --git a/assembly-package/config/exchangis.properties b/assembly-package/config/exchangis.properties deleted file mode 100644 index e69de29bb..000000000 diff --git a/assembly-package/pom.xml b/assembly-package/pom.xml index d8b300d5b..45315a5fe 100644 --- a/assembly-package/pom.xml +++ b/assembly-package/pom.xml @@ -21,7 +21,7 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 assembly-package diff --git a/assembly-package/sbin/common.sh b/assembly-package/sbin/common.sh index 03d4e4666..8ee615b64 100644 --- a/assembly-package/sbin/common.sh +++ b/assembly-package/sbin/common.sh @@ -16,4 +16,4 @@ # declare -A MODULE_MAIN_CLASS -MODULE_MAIN_CLASS["exchangis-server"]="com.webank.wedatasphere.exchangis.server.boot.ExchangisServerApplication" +MODULE_MAIN_CLASS["dss-exchangis-main-server-dev"]="com.webank.wedatasphere.exchangis.server.boot.ExchangisServerApplication" diff --git a/assembly-package/sbin/daemon.sh b/assembly-package/sbin/daemon.sh index d86df111a..a21ccfab6 100644 --- a/assembly-package/sbin/daemon.sh +++ b/assembly-package/sbin/daemon.sh @@ -24,7 +24,7 @@ else fi usage(){ - echo "Usage is [start|stop|restart {service}]" + echo "Usage is [start|stop|restart {server}]" } start(){ diff --git a/assembly-package/sbin/env.properties b/assembly-package/sbin/env.properties index c849c950a..f849b4fa9 100644 --- a/assembly-package/sbin/env.properties +++ b/assembly-package/sbin/env.properties @@ -1,4 +1,4 @@ -EXCHANGIS_CONF_PATH="/appcom/config/exchangis-config" -EXCHANGIS_LOG_PATH="/appcom/logs/exchangis-log" -MODULE_DEFAULT_PREFIX="exchangis-server" -MODULE_DEFAULT_SUFFIX="" +EXCHANGIS_CONF_PATH=/appcom/config/exchangis-config/background +EXCHANGIS_LOG_PATH=/appcom/logs/exchangis/background +MODULE_DEFAULT_PREFIX="dss-exchangis-main-" +MODULE_DEFAULT_SUFFIX="-dev" diff --git a/assembly-package/sbin/install.sh b/assembly-package/sbin/install.sh index 16f453870..2d362038c 100644 --- a/assembly-package/sbin/install.sh +++ b/assembly-package/sbin/install.sh @@ -123,57 +123,36 @@ interact_echo(){ done } +# Initalize database init_database(){ -BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties" -# Start to initalize database -if [ "x${SQL_SOURCE_PATH}" != "x" ] && [ -f "${SQL_SOURCE_PATH}" ]; then - `mysql --version >/dev/null 2>&1` - interact_echo "Do you want to initalize database with sql?" - if [ $? == 0 ]; then - LOG INFO "\033[1m Scan out mysql command, so begin to initalize the database\033[0m" - #interact_echo "Do you want to initalize database with sql: [${SQL_SOURCE_PATH}]?" - #if [ $? == 0 ]; then + BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties" + if [ "x${SQL_SOURCE_PATH}" != "x" ] && [ -f "${SQL_SOURCE_PATH}" ]; then + `mysql --version >/dev/null 2>&1` DATASOURCE_URL="jdbc:mysql:\/\/${MYSQL_HOST}:${MYSQL_PORT}\/${DATABASE}\?useSSL=false\&characterEncoding=UTF-8\&allowMultiQueries=true" - mysql -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} --default-character-set=utf8 -e \ - "CREATE DATABASE IF NOT EXISTS ${DATABASE}; USE ${DATABASE}; source ${SQL_SOURCE_PATH};" - #sed -ri "s![#]?(DB_HOST=)\S*!\1${HOST}!g" ${BOOTSTRAP_PROP_FILE} - #sed -ri "s![#]?(DB_PORT=)\S*!\1${PORT}!g" ${BOOTSTRAP_PROP_FILE} sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.username=)\S*!\1${MYSQL_USERNAME}!g" ${BOOTSTRAP_PROP_FILE} sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.password=)\S*!\1${MYSQL_PASSWORD}!g" ${BOOTSTRAP_PROP_FILE} sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.url=)\S*!\1${DATASOURCE_URL}!g" ${BOOTSTRAP_PROP_FILE} - #fi - fi -fi + interact_echo "Do you want to initalize database with sql: [${SQL_SOURCE_PATH}]?" + if [ $? == 0 ]; then + LOG INFO "\033[1m Scan out mysql command, so begin to initalize the database\033[0m" + mysql -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} --default-character-set=utf8 -e \ + "CREATE DATABASE IF NOT EXISTS ${DATABASE}; USE ${DATABASE}; source ${SQL_SOURCE_PATH};" + fi + fi } init_properties(){ -BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties" -APPLICATION_YML="${CONF_PATH}/application-exchangis.yml" -# Start to initalize propertis - #interact_echo "Do you want to initalize exchangis-server.properties?" - #if [ $? == 0 ]; then - - LINKIS_GATEWAY_URL="http:\/\/${LINKIS_GATEWAY_HOST}:${LINKIS_GATEWAY_PORT}\/" - - if [ "x${LINKIS_SERVER_URL}" == "x" ]; then - LINKIS_SERVER_URL="http://127.0.0.1:3306" - fi - if [ "x${LINKIS_SERVER_URL}" == "x" ]; then - LINKIS_SERVER_URL="http://127.0.0.1:3306" - fi + BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties" + APPLICATION_YML="${CONF_PATH}/application-exchangis.yml" + LINKIS_GATEWAY_URL="http:\/\/${LINKIS_GATEWAY_HOST}:${LINKIS_GATEWAY_PORT}\/" + if [ "x${LINKIS_SERVER_URL}" == "x" ]; then + LINKIS_SERVER_URL="http://127.0.0.1:9001" + fi - sed -ri "s![#]?(wds.linkis.gateway.ip=)\S*!\1${LINKIS_GATEWAY_HOST}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(wds.linkis.gateway.port=)\S*!\1${LINKIS_GATEWAY_PORT}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(wds.linkis.gateway.url=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(wds.exchangis.datasource.client.serverurl=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(wds.exchangis.client.linkis.server-url=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE} - #sed -ri "s![#]?(wds.exchangis.datasource.client.authtoken.key=)\S*!\1${LINKIS_TOKEN}!g" ${BOOTSTRAP_PROP_FILE} - #sed -ri "s![#]?(wds.exchangis.datasource.client.authtoken.value=)\S*!\1${LINKIS_TOKEN}!g" ${BOOTSTRAP_PROP_FILE} - #sed -ri "s![#]?(wds.exchangis.client.linkis.token.value=)\S*!\1${LINKIS_TOKEN}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(wds.linkis.gateway.port=)\S*!\1${LINKIS_GATEWAY_PORT}!g" ${BOOTSTRAP_PROP_FILE} - sed -ri "s![#]?(port: )\S*!\1${EXCHANGIS_PORT}!g" ${APPLICATION_YML} - sed -ri "s![#]?(defaultZone: )\S*!\1${EUREKA_URL}!g" ${APPLICATION_YML} - #fi + sed -ri "s![#]?(wds.exchangis.datasource.client.serverurl=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE} + sed -ri "s![#]?(wds.exchangis.client.linkis.server-url=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE} + sed -ri "s![#]?(port: )\S*!\1${EXCHANGIS_PORT}!g" ${APPLICATION_YML} + sed -ri "s![#]?(defaultZone: )\S*!\1${EUREKA_URL}!g" ${APPLICATION_YML} } install_modules(){ diff --git a/assembly-package/sbin/launcher.sh b/assembly-package/sbin/launcher.sh index db6031428..50a79d279 100644 --- a/assembly-package/sbin/launcher.sh +++ b/assembly-package/sbin/launcher.sh @@ -86,8 +86,6 @@ load_env_definitions(){ fi } - - construct_java_command(){ verify_java_env if [[ "x${EXCHANGIS_CONF_PATH}" == "x" ]]; then @@ -114,7 +112,7 @@ construct_java_command(){ mkdir -p ${EXCHANGIS_PID_PATH} local classpath=${EXCHANGIS_CONF_PATH}":." local opts="" - classpath=${EXCHANGIS_LIB_PATH}/exchangis-server/*":"${classpath} + classpath=${EXCHANGIS_LIB_PATH}/"exchangis-server/*:"${classpath} LOG INFO "classpath:"${classpath} if [[ "x${EXCHANGIS_JAVA_OPTS}" == "x" ]]; then # Use G1 garbage collector @@ -175,7 +173,7 @@ wait_for_startup(){ return 0 fi sleep ${SLEEP_TIMEREVAL_S} - now_s=`date '+%s'` #计算当前时间时间戳 + now_s=`date '+%s'` done return 1 } @@ -204,14 +202,17 @@ launcher_start(){ fi construct_java_command $1 $2 # Execute + echo ${EXEC_JAVA} LOG INFO ${EXEC_JAVA} nohup ${EXEC_JAVA} >/dev/null 2>&1 & LOG INFO "Launcher: waiting [ $1 ] to start complete ..." wait_for_startup 20 $1 $2 if [[ $? -eq 0 ]]; then LOG INFO "Launcher: [ $1 ] start success" + LOG INFO ${EXCHANGIS_CONF_PATH} APPLICATION_YML="${EXCHANGIS_CONF_PATH}/application-exchangis.yml" EUREKA_URL=`cat ${APPLICATION_YML} | grep Zone | sed -n '1p'` + echo "${EUREKA_URL}" LOG INFO "Please check exchangis server in EUREKA_ADDRESS: ${EUREKA_URL#*:} " else LOG ERROR "Launcher: [ $1 ] start fail over 20 seconds, please retry it" @@ -249,4 +250,4 @@ launcher_stop(){ LOG ERROR "Launcher: [ $1 ] stop exceeded over 20s " >&2 return 1 fi -} \ No newline at end of file +} diff --git a/db/1.1.1/exchangis_ddl.sql b/db/1.1.1/exchangis_ddl.sql new file mode 100644 index 000000000..1002aa86b --- /dev/null +++ b/db/1.1.1/exchangis_ddl.sql @@ -0,0 +1,88 @@ +-- exchangis_job_func definition +DROP TABLE IF EXISTS `exchangis_job_func`; +CREATE TABLE `exchangis_job_func` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `func_type` varchar(50) NOT NULL, + `func_name` varchar(100) NOT NULL, + `tab_name` varchar(50) NOT NULL COMMENT 'Tab', + `name_dispaly` varchar(100) DEFAULT NULL, + `param_num` int(11) DEFAULT '0', + `ref_name` varchar(100) DEFAULT NULL, + `description` varchar(200) DEFAULT NULL, + `modify_time` datetime DEFAULT NULL, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `job_func_tab_name_idx` (`tab_name`,`func_name`) +) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8; + +-- exchangis_job_func_params definition +DROP TABLE IF EXISTS `exchangis_job_func_params`; +CREATE TABLE IF NOT EXISTS `exchangis_job_func_params`( + `func_id` INT(11) NOT NULL, + `param_name` VARCHAR(100) NOT NULL, + `order` INT(11) DEFAULT 0, + `name_display` VARCHAR(100), + `create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(`func_id`, `param_name`) +)Engine=InnoDB DEFAULT CHARSET=utf8; + +-- exchangis_job_param_config definition +DROP TABLE IF EXISTS `exchangis_job_param_config`; +CREATE TABLE `exchangis_job_param_config` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `config_key` varchar(64) NOT NULL, + `config_name` varchar(64) NOT NULL, + `config_direction` varchar(16) DEFAULT NULL, + `type` varchar(32) NOT NULL, + `ui_type` varchar(32) DEFAULT NULL, + `ui_field` varchar(64) DEFAULT NULL, + `ui_label` varchar(32) DEFAULT NULL, + `unit` varchar(32) DEFAULT NULL, + `required` bit(1) DEFAULT b'0', + `value_type` varchar(32) DEFAULT NULL, + `value_range` varchar(255) DEFAULT NULL, + `default_value` varchar(255) DEFAULT NULL, + `validate_type` varchar(64) DEFAULT NULL, + `validate_range` varchar(64) DEFAULT NULL, + `validate_msg` varchar(255) DEFAULT NULL, + `is_hidden` bit(1) DEFAULT NULL, + `is_advanced` bit(1) DEFAULT NULL, + `source` varchar(255) DEFAULT NULL, + `level` tinyint(4) DEFAULT NULL, + `treename` varchar(32) DEFAULT NULL, + `sort` int(11) DEFAULT NULL, + `description` varchar(255) DEFAULT NULL, + `status` tinyint(4) DEFAULT NULL, + `ref_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=32 DEFAULT CHARSET=utf8; + +-- exchangis_engine_settings definition +DROP TABLE IF EXISTS `exchangis_engine_settings`; +CREATE TABLE `exchangis_engine_settings` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `engine_name` varchar(50) NOT NULL, + `engine_desc` varchar(500) NOT NULL, + `engine_settings_value` text, + `engine_direction` varchar(255) NOT NULL, + `res_loader_class` varchar(255), + `res_uploader_class` varchar(255), + `modify_time` datetime DEFAULT NULL, + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `engine_setting_idx` (`engine_name`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- exchangis_job_transform_rule +DROP TABLE IF EXISTS `exchangis_job_transform_rule`; +CREATE TABLE `exchangis_job_transform_rule` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `rule_name` varchar(100) NOT NULL DEFAULT 'transform_rule', + `rule_type` varchar(64) NOT NULL DEFAULT 'DEF', + `rule_source` varchar(600) DEFAULT '{}', + `data_source_type` varchar(64) NOT NULL, + `engine_type` varchar(32), + `direction` varchar(32) NOT NULL DEFAULT 'NONE', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file diff --git a/db/1.1.1/exchangis_dml.sql b/db/1.1.1/exchangis_dml.sql new file mode 100644 index 000000000..3e546d667 --- /dev/null +++ b/db/1.1.1/exchangis_dml.sql @@ -0,0 +1,79 @@ +-- job_func records +INSERT INTO `exchangis_job_func`(func_type,func_name,tab_name,name_dispaly,param_num,ref_name,description,modify_time) VALUES +('TRANSFORM','dx_substr','DATAX',NULL,2,NULL,NULL,NULL) +,('TRANSFORM','dx_pad','DATAX',NULL,3,NULL,NULL,NULL) +,('TRANSFORM','dx_replace','DATAX',NULL,3,NULL,NULL,NULL) +,('VERIFY','like','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','not like','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','>','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','<','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','=','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','!=','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','>=','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('TRANSFORM','dx_precision','DATAX',NULL,1,NULL,NULL,NULL) +; + +-- job_func_params records +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(1, 'startIndex', 'startIndex', 0) ON DUPLICATE KEY UPDATE `name_display` = 'startIndex'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(1, 'length', 'length', 1) ON DUPLICATE KEY UPDATE `name_display` = 'length'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(2, 'padType', 'padType(r or l)', 0) ON DUPLICATE KEY UPDATE `name_display` = 'padType(r or l)'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(2, 'length', 'length', 1) ON DUPLICATE KEY UPDATE `name_display` = 'length'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(2, 'padString', 'padString', 2) ON DUPLICATE KEY UPDATE `name_display` = 'padString'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(3, 'startIndex', 'startIndex', 0) ON DUPLICATE KEY UPDATE `name_display` = 'startIndex'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(3, 'length', 'length', 1) ON DUPLICATE KEY UPDATE `name_display` = 'length'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`, `order`) VALUES(3, 'replaceString', 'replaceString', 2) ON DUPLICATE KEY UPDATE `name_display` = 'replaceString'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(4, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(5, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(6, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(7, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(8, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(9, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; +INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) VALUES(10, 'value', 'value') ON DUPLICATE KEY UPDATE `name_display` = 'value'; + +-- job_param_config records +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('setting.speed.byte','作业速率限制','','DATAX','INPUT','setting.speed.bytes','作业速率限制','Mb/s',1,'NUMBER','','5','REGEX','^[1-9]d*$','作业速率限制输入错误',0,0,'',1,'',1,'',1,NULL) +,('setting.speed.record','作业记录数限制','','DATAX','INPUT','setting.speed.records','作业记录数限制','条/s',1,'NUMBER','','100','REGEX','^[1-9]d*$','作业记录数限制输入错误',0,0,'',1,'',2,'',1,NULL) +,('setting.speed.channel','作业最大并行度','','DATAX','INPUT','setting.max.parallelism','作业最大并行度','个',1,'NUMBER','','1','REGEX','^[1-9]d*$','作业最大并行度输入错误',0,0,'',1,'',3,'',1,NULL) +,('setting.max.memory','作业最大使用内存','','DATAX','INPUT','setting.max.memory','作业最大使用内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]d*$','作业最大使用内存输入错误',0,0,'',1,'',4,'',1,NULL) +,('setting.errorLimit.record','最多错误记录数','','DATAX','INPUT','setting.errorlimit.record','最多错误记录数','条',0,'NUMBER','','','REGEX','^[0-9]d*$','最多错误记录数输入错误',0,0,'',1,'',5,'',1,NULL) +,('setting.max.parallelism','作业最大并行数','','SQOOP','INPUT','setting.max.parallelism','作业最大并行数','个',1,'NUMBER','','1','REGEX','^[1-9]d*$','作业最大并行数输入错误',0,0,'',1,'',1,'',1,NULL) +,('setting.max.memory','作业最大内存','','SQOOP','INPUT','setting.max.memory','作业最大内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]d*$','作业最大内存输入错误',0,0,'',1,'',2,'',1,NULL) +,('where','WHERE条件','SOURCE','MYSQL','INPUT','where','WHERE条件','',0,'VARCHAR','','','REGEX','^[sS]{0,500}$','WHERE条件输入过长',0,0,'',1,'',2,'',1,NULL) +,('writeMode','写入方式','SQOOP-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["OVERWRITE","APPEND"]','OVERWRITE','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) +; +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) +,('writeMode','写入方式','SQOOP-SINK','MYSQL','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","UPDATE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('batchSize','批量大小','DATAX-SINK','ELASTICSEARCH','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]d*$','批量大小输入错误',0,0,'',1,'',1,'',1,NULL) +,('query','query条件','DATAX-SOURCE','MONGODB','INPUT','query','query条件','',0,'VARCHAR','','','REGEX','^[sS]{0,500}$','query条件输入过长',0,0,'',1,'',2,'',1,NULL) +,('writeMode','写入方式','DATAX-SINK','MONGODB','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","REPLACE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('batchSize','批量大小','DATAX-SINK','MONGODB','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL) +,('transferMode','传输方式','DATAX-SOURCE','HIVE','OPTION','transferMode','传输方式','',1,'OPTION','["二进制","记录"]','二进制','','','该传输方式不可用',0,0,'',1,'',1,'',1,NULL) +,('nullFormat','空值字符','DATAX-SOURCE','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +,('writeMode','写入方式','DATAX-SINK','MYSQL','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","UPDATE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('writeMode','写入方式','DATAX-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["append","truncate"]','append','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +; +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('nullFormat','空值字符','DATAX-SINK','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +,('nullFormat','空值字符','DATAX-SINK','ELASTICSEARCH','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +; +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('where','WHERE条件','SOURCE','ORACLE','INPUT','where','WHERE条件',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,NULL,1,'',2,NULL,1,NULL) +,('writeMode','写入方式','DATAX-SINK','ORACLE','OPTION','writeMode','写入方式',NULL,1,'OPTION','["INSERT","UPDATE"]','INSERT',NULL,NULL,'写入方式输入错误',0,0,NULL,1,NULL,1,NULL,1,NULL) +; + +-- engine_settings records +INSERT INTO `exchangis_engine_settings` (id, engine_name, engine_desc, engine_settings_value, engine_direction, res_loader_class, res_uploader_class, modify_time, create_time) VALUES +(1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL, '2022-08-09 18:20:51.0'), +(2, 'sqoop', 'hadoop tool', '{}', 'mysql->hive,hive->mysql', '', NULL, NULL, '2022-08-09 18:20:51.0'); + +-- exchangis_job_transform_rule records +INSERT INTO `exchangis_job_transform_rule` (rule_name,rule_type,rule_source,data_source_type,engine_type,direction) VALUES +('es_with_post_processor','DEF','{"types": ["MAPPING", "PROCESSOR"]}','ELASTICSEARCH',NULL,'SINK') +,('es_fields_not_editable','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false}','ELASTICSEARCH',NULL,'SINK') +,('hive_sink_not_access','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false, "fieldAddEnable": false}','HIVE',NULL,'SINK') +,('mongo_field_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH"}','MONGODB',NULL,'SINK') +,('mysql_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": false}','MYSQL',NULL,'SOURCE') +; \ No newline at end of file diff --git a/db/1.1.2/exchangis_ddl.sql b/db/1.1.2/exchangis_ddl.sql new file mode 100644 index 000000000..3609cadfd --- /dev/null +++ b/db/1.1.2/exchangis_ddl.sql @@ -0,0 +1 @@ +ALTER TABLE exchangis_job_entity MODIFY COLUMN name varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL; \ No newline at end of file diff --git a/db/exchangis_ddl.sql b/db/exchangis_ddl.sql index 58f1e1df2..c04796624 100644 --- a/db/exchangis_ddl.sql +++ b/db/exchangis_ddl.sql @@ -14,7 +14,7 @@ CREATE TABLE `exchangis_job_ds_bind` ( DROP TABLE IF EXISTS `exchangis_job_entity`; CREATE TABLE `exchangis_job_entity` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, - `name` varchar(100) NOT NULL, + `name` varchar(255) NOT NULL, `create_time` datetime DEFAULT NULL, `last_update_time` datetime(3) DEFAULT NULL, `engine_type` varchar(45) DEFAULT '', @@ -178,7 +178,7 @@ CREATE TABLE `exchangis_job_func` ( `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `job_func_tab_name_idx` (`tab_name`,`func_name`) -) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8 +) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8; -- exchangis_job_func_params definition DROP TABLE IF EXISTS `exchangis_job_func_params`; diff --git a/db/exchangis_dml.sql b/db/exchangis_dml.sql index 1761fc734..2e6bee29e 100644 --- a/db/exchangis_dml.sql +++ b/db/exchangis_dml.sql @@ -1,18 +1,16 @@ -- job_func records -INSERT INTO exchangis_job_func (func_type,func_name,tab_name,name_dispaly,param_num,ref_name,description,modify_time,create_time) VALUES -('TRANSFORM','dx_substr','DATAX',NULL,2,NULL,NULL,NULL,'2020-04-21 17:35:06.000') -,('TRANSFORM','dx_pad','DATAX',NULL,3,NULL,NULL,NULL,'2020-04-21 17:35:06.000') -,('TRANSFORM','dx_replace','DATAX',NULL,3,NULL,NULL,NULL,'2020-04-21 17:35:06.000') -,('VERIFY','like','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') -,('VERIFY','not like','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') -,('VERIFY','>','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') -,('VERIFY','<','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') -,('VERIFY','=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') -,('VERIFY','!=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') -,('VERIFY','>=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000') -; -INSERT INTO exchangis_job_func (func_type,func_name,tab_name,name_dispaly,param_num,ref_name,description,modify_time,create_time) VALUES -('TRANSFORM','dx_precision','DATAX',NULL,1,NULL,NULL,NULL,'2020-04-21 17:35:06.000') +INSERT INTO `exchangis_job_func`(func_type,func_name,tab_name,name_dispaly,param_num,ref_name,description,modify_time) VALUES +('TRANSFORM','dx_substr','DATAX',NULL,2,NULL,NULL,NULL) +,('TRANSFORM','dx_pad','DATAX',NULL,3,NULL,NULL,NULL) +,('TRANSFORM','dx_replace','DATAX',NULL,3,NULL,NULL,NULL) +,('VERIFY','like','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','not like','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','>','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','<','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','=','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','!=','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('VERIFY','>=','DATAX',NULL,1,'dx_filter',NULL,NULL) +,('TRANSFORM','dx_precision','DATAX',NULL,1,NULL,NULL,NULL) ; -- job_func_params records @@ -34,49 +32,47 @@ INSERT INTO `exchangis_job_func_params`(`func_id`, `param_name`, `name_display`) -- job_param_config records INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES -('setting.speed.byte','作业速率限制','','DATAX','INPUT','setting.speed.bytes','作业速率限制','Mb/s',1,'NUMBER','','5','REGEX','^[1-9]d*$','作业速率限制输入错误',0,0,'',1,'',1,'',1,NULL) -,('setting.speed.record','作业记录数限制','','DATAX','INPUT','setting.speed.records','作业记录数限制','条/s',1,'NUMBER','','100','REGEX','^[1-9]d*$','作业记录数限制输入错误',0,0,'',1,'',2,'',1,NULL) -,('setting.speed.channel','作业最大并行度','','DATAX','INPUT','setting.max.parallelism','作业最大并行度','个',1,'NUMBER','','1','REGEX','^[1-9]d*$','作业最大并行度输入错误',0,0,'',1,'',3,'',1,NULL) -,('setting.max.memory','作业最大使用内存','','DATAX','INPUT','setting.max.memory','作业最大使用内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]d*$','作业最大使用内存输入错误',0,0,'',1,'',4,'',1,NULL) -,('setting.errorLimit.record','最多错误记录数','','DATAX','INPUT','setting.errorlimit.record','最多错误记录数','条',0,'NUMBER','','','REGEX','^[0-9]d*$','最多错误记录数输入错误',0,0,'',1,'',5,'',1,NULL) -,('setting.max.parallelism','作业最大并行数','','SQOOP','INPUT','setting.max.parallelism','作业最大并行数','个',1,'NUMBER','','1','REGEX','^[1-9]\d*$','作业最大并行数输入错误',0,0,'',1,'',1,'',1,NULL) -,('setting.max.memory','作业最大内存','','SQOOP','INPUT','setting.max.memory','作业最大内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\d*$','作业最大内存输入错误',0,0,'',1,'',2,'',1,NULL) -,('where','WHERE条件','SOURCE','MYSQL','INPUT','where','WHERE条件','',0,'VARCHAR','','','REGEX','^[sS]{0,500}$','WHERE条件输入过长',0,0,'',1,'',2,'',1,NULL) +('setting.speed.byte','作业速率限制','','DATAX','INPUT','setting.speed.bytes','作业速率限制','Mb/s',1,'NUMBER','','5','REGEX','^[1-9]\\d*$','作业速率限制输入错误',0,0,'',1,'',1,'',1,NULL) +,('setting.speed.record','作业记录数限制','','DATAX','INPUT','setting.speed.records','作业记录数限制','条/s',1,'NUMBER','','100','REGEX','^[1-9]\\d*$','作业记录数限制输入错误',0,0,'',1,'',2,'',1,NULL) +,('setting.speed.channel','作业最大并行度','','DATAX','INPUT','setting.max.parallelism','作业最大并行度','个',1,'NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行度输入错误',0,0,'',1,'',3,'',1,NULL) +,('setting.max.memory','作业最大使用内存','','DATAX','INPUT','setting.max.memory','作业最大使用内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大使用内存输入错误',0,0,'',1,'',4,'',1,NULL) +,('setting.errorLimit.record','最多错误记录数','','DATAX','INPUT','setting.errorlimit.record','最多错误记录数','条',0,'NUMBER','','','REGEX','^[0-9]\\d*$','最多错误记录数输入错误',0,0,'',1,'',5,'',1,NULL) +,('setting.max.parallelism','作业最大并行数','','SQOOP','INPUT','setting.max.parallelism','作业最大并行数','个',1,'NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行数输入错误',0,0,'',1,'',1,'',1,NULL) +,('setting.max.memory','作业最大内存','','SQOOP','INPUT','setting.max.memory','作业最大内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大内存输入错误',0,0,'',1,'',2,'',1,NULL) +,('where','WHERE条件','SOURCE','MYSQL','INPUT','where','WHERE条件','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,'',1,'',2,'',1,NULL) ,('writeMode','写入方式','SQOOP-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["OVERWRITE","APPEND"]','OVERWRITE','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) -,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) +,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) ; INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES -('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) +('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) ,('writeMode','写入方式','SQOOP-SINK','MYSQL','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","UPDATE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) -,('batchSize','批量大小','DATAX-SINK','ELASTICSEARCH','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\d*$','批量大小输入错误',0,0,'',1,'',1,'',1,NULL) -,('query','query条件','DATAX-SOURCE','MONGODB','INPUT','query','query条件','',0,'VARCHAR','','','REGEX','^[sS]{0,500}$','query条件输入过长',0,0,'',1,'',2,'',1,NULL) +,('batchSize','批量大小','DATAX-SINK','ELASTICSEARCH','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',1,'',1,NULL) +,('query','query条件','DATAX-SOURCE','MONGODB','INPUT','query','query条件','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,500}$','query条件输入过长',0,0,'',1,'',2,'',1,NULL) ,('writeMode','写入方式','DATAX-SINK','MONGODB','OPTION','writeMode','写入方式','',1,'OPTION','["insert","replace"]','insert','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) -,('batchSize','批量大小','DATAX-SINK','MONGODB','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL) +,('batchSize','批量大小','DATAX-SINK','MONGODB','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL) ,('transferMode','传输方式','DATAX-SOURCE','HIVE','OPTION','transferMode','传输方式','',1,'OPTION','["二进制","记录"]','二进制','','','该传输方式不可用',0,0,'',1,'',1,'',1,NULL) -,('nullFormat','空值字符','DATAX-SOURCE','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +,('nullFormat','空值字符','DATAX-SOURCE','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) ,('writeMode','写入方式','DATAX-SINK','MYSQL','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","UPDATE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) ,('writeMode','写入方式','DATAX-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["append","truncate"]','append','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) ; INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES -('nullFormat','空值字符','DATAX-SINK','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) -,('nullFormat','空值字符','DATAX-SINK','ELASTICSEARCH','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) -; - -INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES -('where','WHERE条件','SOURCE','ORACLE','INPUT','where','WHERE条件',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,NULL,1,'',2,NULL,1,NULL) +('nullFormat','空值字符','DATAX-SINK','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +,('nullFormat','空值字符','DATAX-SINK','ELASTICSEARCH','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +,('where','WHERE条件','SOURCE','ORACLE','INPUT','where','WHERE条件',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,NULL,1,'',2,NULL,1,NULL) ,('writeMode','写入方式','DATAX-SINK','ORACLE','OPTION','writeMode','写入方式',NULL,1,'OPTION','["INSERT","UPDATE"]','INSERT',NULL,NULL,'写入方式输入错误',0,0,NULL,1,NULL,1,NULL,1,NULL) ; -- engine_settings records -INSERT INTO `exchangis_engine_settings` (id, engine_name, engine_desc, engine_settings_value, engine_direction, res_loader_class, res_uploader_class, modify_time, create_time) VALUES +INSERT INTO `exchangis_engine_settings` (id, engine_name, engine_desc, engine_settings_value, engine_direction, res_loader_class, res_uploader_class, modify_time) VALUES (1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL, '2022-08-09 18:20:51.0'), -(2, 'sqoop', 'hadoop tool', '{}', 'mysql->hive,hive->mysql', '', NULL, NULL, '2022-08-09 18:20:51.0'); +(2, 'sqoop', 'hadoop tool', '{}', 'mysql->hive,hive->mysql', '', NULL, NULL); -INSERT INTO `exchangis_job_transform_rule` (rule_name,rule_type,rule_source,data_source_type,engine_type,direction,create_time) VALUES -('es_with_post_processor','DEF','{"types": ["MAPPING", "PROCESSOR"]}','ELASTICSEARCH',NULL,'SINK','2022-09-18 23:36:42.000') -,('es_fields_not_editable','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false}','ELASTICSEARCH',NULL,'SINK','2022-09-20 16:27:18.000') -,('hive_sink_not_access','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false, "fieldAddEnable": false}','HIVE',NULL,'SINK','2022-09-20 21:46:05.000') -,('mongo_field_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH"}','MONGODB',NULL,'SINK','2022-09-20 21:55:05.000') -,('mysql_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": false}','MYSQL',NULL,'SOURCE','2022-09-21 21:55:05.000') +-- exchangis_job_transform_rule records +INSERT INTO `exchangis_job_transform_rule` (rule_name,rule_type,rule_source,data_source_type,engine_type,direction) VALUES +('es_with_post_processor','DEF','{"types": ["MAPPING", "PROCESSOR"]}','ELASTICSEARCH',NULL,'SINK') +,('es_fields_not_editable','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false}','ELASTICSEARCH',NULL,'SINK') +,('hive_sink_not_access','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false, "fieldAddEnable": false}','HIVE',NULL,'SINK') +,('mongo_field_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH"}','MONGODB',NULL,'SINK') +,('mysql_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": false}','MYSQL',NULL,'SOURCE') ; diff --git a/docs/en_US/ch1/component_upgrade_en.md b/docs/en_US/ch1/component_upgrade_en.md index 7f9e69d95..c3aa23379 100644 --- a/docs/en_US/ch1/component_upgrade_en.md +++ b/docs/en_US/ch1/component_upgrade_en.md @@ -1,7 +1,7 @@ # Exchangis Component Upgrade Documentation -This article mainly introduces the upgrade steps for adapting DSS1.1.0 and Linkis1.1.1 on the basis of the original installation of the Exchangis service. The biggest difference between the Exchangis1.0.0 and the Exchangis1.0.0-rc1 version is the installation of the ExchangisAppconn, which needs to be replaced by the entire Exchangisappconn. and load### 1.升级Exchangis前的工作 -Before you upgrade Exchangis, please follow the[DSS1.1.0Install and deploy documentation](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_deploy_cn.md) -and [Linkis1.1.1Install and deploy documentation](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_deploy_cn.md)Complete the installation and upgrade of DSS and Linkis +This article mainly introduces the upgrade steps for adapting DSS1.1.2 and Linkis1.4.0 on the basis of the original installation of the Exchangis service. The biggest difference between the Exchangis1.0.0 and the Exchangis1.0.0-rc1 version is the installation of the ExchangisAppconn, which needs to be replaced by the entire Exchangisappconn. and load### 1.升级Exchangis前的工作 +Before you upgrade Exchangis, please follow the[DSS1.1.2Install and deploy documentation](https://github.com/WeBankFinTech/DataSphereStudio-Doc/tree/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2) +and [Linkis1.4.0Install and deploy documentation](https://linkis.staged.apache.org/zh-CN/docs/1.4.0/deployment/deploy-quick)Complete the installation and upgrade of DSS and Linkis ### 2.Exchangis upgrade steps diff --git a/docs/en_US/ch1/exchangis_datasource_en.md b/docs/en_US/ch1/exchangis_datasource_en.md index 9dc125718..ffbffdf98 100644 --- a/docs/en_US/ch1/exchangis_datasource_en.md +++ b/docs/en_US/ch1/exchangis_datasource_en.md @@ -106,7 +106,7 @@ Batch import and export of datasource configuration. Description: -The background needs to integrate it with the labeling function of Linkis1.1.1, and give the datasource a labeling relationship. +The background needs to integrate it with the labeling function of Linkis1.4.0, and give the datasource a labeling relationship. Process design: diff --git a/docs/en_US/ch1/exchangis_deploy_en.md b/docs/en_US/ch1/exchangis_deploy_en.md index faecef887..65ecb03f1 100644 --- a/docs/en_US/ch1/exchangis_deploy_en.md +++ b/docs/en_US/ch1/exchangis_deploy_en.md @@ -13,22 +13,22 @@ Exchangis installation is mainly divided into the following four steps ： | Dependent components | Must be installed | Install through train | |------------------------------------------------------------------------------| ------ | --------------- | -| MySQL (5.5+) | yes | [How to install mysql](https://www.runoob.com/mysql/mysql-install.html) | -| JDK (1.8.0_141) | yes | [How to install JDK](https://www.runoob.com/java/java-environment-setup.html) | -| Hadoop(2.7.2，Other versions of Hadoop need to compile Linkis by themselves.) | yes | [Hadoop stand-alone deployment](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) ；[Hadoop distributed deployment](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | -| Hive(2.3.3，Other versions of Hive need to compile Linkis by themselves.) | yes | [Hive quick installation](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | +| JDK (1.8.0_141) | yes | [How to install JDK](https://www.oracle.com/java/technologies/downloads/) | +| MySQL (5.5+) | yes | [How to install mysql](https://www.mysql.com/downloads/) | +| Hadoop(3.3.4，Other versions of Hadoop need to compile Linkis by themselves.) | yes | [Hadoop deployment](https://www.apache.org/dyn/closer.cgi/hadoop/common/hadoop-3.3.4/hadoop-3.3.4.tar.gz) | +| Hive(2.3.3，Other versions of Hive need to compile Linkis by themselves.) | yes | [Hive quick installation](https://www.apache.org/dyn/closer.cgi/hive/) | | SQOOP (1.4.6) | yes | [How to install Sqoop](https://sqoop.apache.org/docs/1.4.6/SqoopUserGuide.html) | -| DSS1.1.0 | yes | [How to install DSS](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) | -| Linkis1.1.1 | yes | [How to install Linkis](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | -| Nginx | yes | [How to install Nginx](http://nginx.org/en/linux_packages.html) | +| DSS1.1.2 | yes | [How to install DSS](https://github.com/WeBankFinTech/DataSphereStudio-Doc/tree/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2) | +| Linkis1.4.0 | yes | [How to install Linkis](https://linkis.apache.org/zh-CN/docs/1.4.0/deployment/deploy-quick) | +| Nginx | yes | [How to install Nginx](http://nginx.org/) | Underlying component checking -$\color{#FF0000}{Note: be sure to reinstall dss1.1.0, and the linkis version must be greater than 1.1.1. Please recompile linkis and use the package released on June 15th }$ +$\color{#FF0000}{Note: be sure to reinstall dss1.1.2, and linkis1.4.0. Please recompile linkis and use the package released on June 15th }$ -[linkis1.1.1 code address ](https://github.com/apache/incubator-linkis/tree/release-1.1.1) +[linkis1.4.0 code address ](https://github.com/apache/incubator-linkis/tree/release-1.4.0) -[DSS1.1.0 code address ](https://github.com/WeBankFinTech/DataSphereStudio/tree/dev-1.1.0) +[DSS1.1.2 code address ](https://github.com/WeBankFinTech/DataSphereStudio/tree/dev-1.1.2) datasource enabled @@ -55,11 +55,11 @@ INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_t INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_type_id`, `parameter`, `create_time`, `create_user`, `modify_time`, `modify_user`) VALUES ('开发环境UAT', '开发环境UAT', 4, '{"uris":"thrift://${HIVE_METADATA_IP}:${HIVE_METADATA_PORT}", "hadoopConf":{"hive.metastore.execute.setugi":"true"}}', now(), NULL, now(), NULL); ``` -If the hive data source needs kerberos authentication when deployed, you need to specify a parameter keyTab in the parameter field of the Linkis_ps_dm_datasource_env table, and the way to obtain its value can be seen: [Setting and authenticating hive data source in linkis](https://linkis.apache.org/zh-CN/docs/1.1.1/deployment/start-metadatasource). +If the hive data source needs kerberos authentication when deployed, you need to specify a parameter keyTab in the parameter field of the Linkis_ps_dm_datasource_env table, and the way to obtain its value can be seen: [Setting and authenticating hive data source in linkis](https://linkis.apache.org/zh-CN/docs/latest/auth/token). #### 1.4 Underlying component checking -Please ensure that DSS1.1.0 and Linkis1.1.1 are basically available. HiveQL scripts can be executed in the front-end interface of DSS, and DSS workflows can be created and executed normally. +Please ensure that DSS1.1.2 and Linkis1.4.0 are basically available. HiveQL scripts can be executed in the front-end interface of DSS, and DSS workflows can be created and executed normally. ## 2. Exchangis installation and deployment @@ -67,7 +67,7 @@ Please ensure that DSS1.1.0 and Linkis1.1.1 are basically available. HiveQL scri #### 2.1.1 Download binary package -Download the latest installation package from the Released release of Exchangis [click to jump to the release interface](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.0.0). +Download the latest installation package from the Released release of Exchangis [click to jump to the release interface](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.1.2). #### 2.1.2 Compile and package @@ -94,6 +94,8 @@ Download the latest installation package from the Released release of Exchangis |-- db：Database initialization SQL directory |-- exchangis-extds |-- packages：Exchangis installation package directory + |-- exchangis-extds：exchangis datasource library + |-- lib：library |-- sbin：Script storage directory ``` @@ -159,7 +161,28 @@ DATABASE= Enter `y` to start installing Exchange IS service, or `n` to not install it. -#### 2.5.3 Start service +#### 2.5.3 Change the path of the configuration file and log file + +In the 'env.properties' file in the sbin directory, set the configuration file path and log file path + +```yaml +EXCHANGIS_CONF_PATH="/appcom/config/exchangis-config/background" +EXCHANGIS_LOG_PATH="/appcom/logs/exchangis/background" +MODULE_DEFAULT_PREFIX="dss-exchangis-main-" +MODULE_DEFAULT_SUFFIX="-dev" +``` + +EXCHANGIS_CONF_PATH indicates the configuration file path, and EXCHANGIS_LOG_PATH indicates the log file path. If the preceding configurations are used, perform the following operations: + +```yaml +cd {EXCHANGIS_DEPLOY_PATH} +cp -r config /appcom/config/exchangis-config/background +mkdir -p /appcom/logs/exchangis/background +``` + +When the service is started, the configuration file in the corresponding path is used and logs are written to the corresponding path + +#### 2.5.4 Start service Execute the following command to start Exchangis Server: @@ -175,7 +198,7 @@ Execute the following command to start Exchangis Server: After executing the startup script, the following prompt will appear, eureka address will also be typed in the console when starting the service: -![企业微信截图_16532930262583](https://user-images.githubusercontent.com/27387830/169773764-1c5ed6fb-35e9-48cb-bac8-6fa7f738368a.png) +![企业微信截图_16532930262583](../../../images/zh_CN/ch1/register_eureka.png) ### 2.6 Check whether the service started successfully. @@ -205,7 +228,7 @@ Get the compiled dist.zip front-end package from the `web/` path. The acquired front-end package can be placed anywhere on the server. Here, it is recommended that you keep the same directory as the back-end installation address, place it in the same directory and unzip it. -#### 2.7.2 Front-end installation deployment +#### 3.3.4 Front-end installation deployment 1. Decompress front-end installation package diff --git a/docs/en_US/ch1/exchangis_sqoop_deploy_en.md b/docs/en_US/ch1/exchangis_sqoop_deploy_en.md index 60f796121..b84e3f7cf 100644 --- a/docs/en_US/ch1/exchangis_sqoop_deploy_en.md +++ b/docs/en_US/ch1/exchangis_sqoop_deploy_en.md @@ -26,7 +26,7 @@ It is strongly recommended that you use the native Sqoop to perform the test tas ### Prepare installation package #### 1）Download binary package -Exchangis1.0.0 and Linkis 1.1.1 support the mainstream Sqoop versions 1.4.6 and 1.4.7, and later versions may need to modify some codes for recompilation. +Exchangis1.1.2 and Linkis 1.4.0 support the mainstream Sqoop versions 1.4.6 and 1.4.7, and later versions may need to modify some codes for recompilation. [Click to jump to Release interface](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.0.0) diff --git a/docs/zh_CN/ch1/component_upgrade_cn.md b/docs/zh_CN/ch1/component_upgrade_cn.md index fd4501f35..7858b31b3 100644 --- a/docs/zh_CN/ch1/component_upgrade_cn.md +++ b/docs/zh_CN/ch1/component_upgrade_cn.md @@ -1,8 +1,8 @@ Exchangis 升级文档 -本文主要介绍在原有安装Exchangis服务的基础上适配DSS1.1.0和Linkis1.1.1的升级步骤，Exchangis1.0.0相对与Exchangis1.0.0-rc1版本最大的区别在于ExchangisAppconn的安装，需要对整个Exchangisappconn进行重新替换和加载 +本文主要介绍在原有安装Exchangis服务的基础上适配DSS1.1.2和Linkis1.4.0的升级步骤，Exchangis1.1.2相对与Exchangis1.0.0-rc1版本最大的区别在于ExchangisAppconn的安装，需要对整个Exchangisappconn进行重新替换和加载 ### 1.升级Exchangis前的工作 -您在升级Exchangis之前，请按照[DSS1.1.0安装部署文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_deploy_cn.md) -和[Linkis1.1.1安装部署文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_deploy_cn.md)完成DSS和Linkis的安装升级 +您在升级Exchangis之前，请按照[DSS1.1.2安装部署文档](https://github.com/WeBankFinTech/DataSphereStudio-Doc/tree/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2) +和[Linkis1.4.0安装部署文档](https://linkis.staged.apache.org/zh-CN/docs/1.4.0/deployment/deploy-quick)完成DSS和Linkis的安装升级 ### 2.Exchangis升级步骤 @@ -14,7 +14,7 @@ Exchangis 升级文档 ``` #### 2）下载二进制包 -我们提供ExchangisAppconn的升级物料包，您可直接下载使用。[点击跳转 Release 界面](https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeDatasphere/Exchangis/exchangis1.0.0-rc/exchangis-appconn.zip) +我们提供ExchangisAppconn的升级物料包，您可直接下载使用。[点击跳转 Release 界面](https://osp-1257653870.cos.ap-guangzhou.myqcloud.com/WeDatasphere/Exchangis/exchangis1.0.0-rc/exchangis-appconn.zip)（TODO：待发版时更新） #### 3） 编译打包 @@ -73,7 +73,7 @@ lib/exchangis-server ``` 再通过以下命令完成 Exchangis Server 的更新重启： - + ```shell script ./sbin/daemon.sh restart server ``` @@ -94,4 +94,4 @@ wds.linkis-session.ticket.key=bdp-user-ticket-id ![image](https://user-images.githubusercontent.com/27387830/169782337-678f2df0-080a-495a-b59f-a98c5a427cf8.png) - 更多使用操作可参照[Exchangis1.0用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.1-rc/docs/zh_CN/ch1/exchangis_user_manual_cn.md) + 更多使用操作可参照[Exchangis用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.1.2/docs/zh_CN/ch1/exchangis_user_manual_cn.md) diff --git a/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md b/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md new file mode 100644 index 000000000..81c1a821d --- /dev/null +++ b/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md @@ -0,0 +1,72 @@ +# DataX 引擎使用文档 +### 环境准备 +DataX引擎是执行Exchangis数据同步任务不可或缺的组件，只有安装部署完成DataX引擎才能够成功执行数据同步任务。同时，确保所部署的机器上有安装DataX。 + +您在安装部署DataX引擎之前，请按照[Exchangis1.1.2安装部署文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.1.2/docs/zh_CN/ch1/exchangis_deploy_cn.md)（TODO 待修改连接地址）安装完成Exchangis1.1.2及相关组件的安装，并确保工程基本功能可用。 + +强烈建议您在执行DataX任务之前，先在该节点使用原生的DataX执行测试任务，以检测该节点环境是否正常。 + +| 环境变量名 | 环境变量内容 | 备注 | +| :----: | :----: |-------| +| JAVA_HOME | JDK安装路径 | 必须 | +| DataX_HOME | DataX安装路径 | 非必须 | +| DataX_CONF_DIR | DataX配置路径 | 非必须 | + +### 安装包准备 +#### 1）下载二进制包 + +Exchangis1.1.2和Linkis 1.4.0支持的主流DataX版本1.4.6与1.4.7，更高版本可能需要修改部分代码重新编译。 + +[点击跳转 Release 界面](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.1.2)（TODO 待修改连接地址） + +#### 2） 编译打包 +如果您想自己开发和编译datax引擎，具体编译步骤如下： + +1、克隆Exchangis的代码 + +2、在exchangis-plugins模块下，找到datax引擎，单独编译datax，操作如下 + +``` +cd {EXCHANGIS_CODE_HOME}/exchangis-plugins/engine/datax +mvn clean install +``` +然后会在该路径下找到datax引擎安装包 +``` +{EXCHANGIS_CODE_HOME}/exchangis-plugins/datax/target/out +``` + + +### 开始部署 +#### 1）DataX引擎安装 +1、拿到打包出来的datax物料包，目录结构为： + +```shell +datax +-- dist +-- plugin +``` + +2、放置到linkis安装路径的如下目录 + +```shell +cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins +``` +（注意，看当前datax引擎对哪些用户有权限，一般都为hadoop用户组和hadoop用户） + + +#### 2）重启linkis-engineplugin服务使datax引擎生效 +新加入linkis的引擎都要重启linkis的engineplugin服务才会生效，重启脚本为linkis安装目录下的./sbin/linkis-daemon.sh，具体步骤如下 +```shell +cd {LINKIS_INSTALL_HOME}/links/sbin/ +./linkis-daemon.sh restart cg-engineplugin +``` +待服务启动成功，在linkis数据库中校验datax引擎是否安装完毕 + +```shell +select * from linkis_cg_engine_conn_plugin_bml_resources where engine_conn_type='datax'; +``` + +至此，datax安装部署就完成了。 + +engineplugin更详细的介绍可以参看下面的文章。 +https://linkis.apache.org/zh-CN/docs/latest/deployment/engine-conn-plugin-installation/ \ No newline at end of file diff --git a/docs/zh_CN/ch1/exchangis_deploy_cn.md b/docs/zh_CN/ch1/exchangis_deploy_cn.md index 1ff34d0d2..b25e49a77 100644 --- a/docs/zh_CN/ch1/exchangis_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_deploy_cn.md @@ -6,6 +6,7 @@ Exchangis 的安装，主要分为以下四步： 2. Exchangis 安装部署 3. DSS ExchangisAppConn 安装部署 4. Linkis Sqoop 引擎安装部署 +5. Linkis DataX 引擎安装部署 ## 1. Exchangis 依赖环境准备 @@ -13,26 +14,26 @@ Exchangis 的安装，主要分为以下四步： | 依赖的组件 | 是否必装 | 安装直通车 | |---------------------------------------| ------ | --------------- | -| MySQL (5.5+) | 必装 | [如何安装mysql](https://www.runoob.com/mysql/mysql-install.html) | -| JDK (1.8.0_141) | 必装 | [如何安装JDK](https://www.runoob.com/java/java-environment-setup.html) | -| Hadoop(2.7.2，Hadoop 其他版本需自行编译 Linkis) | 必装 | [Hadoop单机部署](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) ；[Hadoop分布式部署](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | -| Hive(2.3.3，Hive 其他版本需自行编译 Linkis) | 必装 | [Hive快速安装](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | +| JDK (1.8.0_141) | 必装 | [如何安装JDK](https://www.oracle.com/java/technologies/downloads/) | +| MySQL (5.5+) | 必装 | [如何安装mysql](https://www.runoob.com/mysql/mysql-install.html) | +| Hadoop(3.3.4，Hadoop 其他版本需自行编译 Linkis) | 必装 | [Hadoop部署](https://www.apache.org/dyn/closer.cgi/hadoop/common/hadoop-3.3.4/hadoop-3.3.4.tar.gz) | +| Hive(2.3.3，Hive 其他版本需自行编译 Linkis) | 必装 | [Hive快速安装](https://www.apache.org/dyn/closer.cgi/hive/) | | SQOOP (1.4.6) | 必装 | [如何安装Sqoop](https://sqoop.apache.org/docs/1.4.6/SqoopUserGuide.html) | -| DSS1.1.0 | 必装 | [如何安装DSS](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) | -| Linkis1.1.1 | 必装 | [如何安装Linkis](https://linkis.apache.org/zh-CN/docs/latest/deployment/quick-deploy) | -| Nginx | 必装 | [如何安装 Nginx](http://nginx.org/en/linux_packages.html) | +| DSS1.1.2 | 必装 | [如何安装DSS](https://github.com/WeBankFinTech/DataSphereStudio-Doc/tree/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2) | +| Linkis1.4.0 | 必装 | [如何安装Linkis](https://linkis.apache.org/zh-CN/docs/1.4.0/deployment/deploy-quick) | +| Nginx | 必装 | [如何安装 Nginx](http://nginx.org/) | 底层依赖组件检查 -$\color{#FF0000}{注意：一定要使用最新版的dss1.1.0，及linkis1.1.1}$。 +注意：一定要使用最新版的dss1.1.2，及linkis1.4.0。 -[linkis1.1.1代码地址](https://github.com/apache/incubator-linkis/tree/release-1.1.1) +[linkis1.4.0代码地址](https://github.com/apache/incubator-linkis/tree/release-1.4.0) -[DSS1.1.0代码地址](https://github.com/WeBankFinTech/DataSphereStudio/tree/dev-1.1.0) +[DSS1.1.2代码地址 ](https://github.com/WeBankFinTech/DataSphereStudio/tree/dev-1.1.2) datasource启用 -linkis的启动脚本中默认不会启动数据源相关的服务两个服务（ps-data-source-manager，ps-metadatamanager）， 如果想使用数据源服务，可以通过如下方式进行开启: 修改$LINKIS_CONF_DIR/linkis-env.sh中的 export ENABLE_METADATA_MANAGER=true值为true。 通过linkis-start-all.sh/linkis-stop-all.sh 进行服务启停时，会进行数据源服务的启动与停止。关于数据源更多详情可参考[数据源功能使用](https://linkis.apache.org/zh-CN/docs/1.1.0/deployment/start-metadatasource) +linkis的启动脚本中默认不会启动数据源相关的服务两个服务（ps-data-source-manager，ps-metadatamanager）， 如果想使用数据源服务，可以通过如下方式进行开启: 修改$LINKIS_CONF_DIR/linkis-env.sh中的 export ENABLE_METADATA_MANAGER=true值为true。 通过linkis-start-all.sh/linkis-stop-all.sh 进行服务启停时，会进行数据源服务的启动与停止。关于数据源更多详情可参考[数据源功能使用](https://linkis.apache.org/zh-CN/docs/1.4.0/user-guide/datasource-manual) #### 1.2 创建 Linux 用户 @@ -57,11 +58,11 @@ INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_t INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_type_id`, `parameter`, `create_time`, `create_user`, `modify_time`, `modify_user`) VALUES ('开发环境UAT', '开发环境UAT', 4, '{"uris":"thrift://${HIVE_METADATA_IP}:${HIVE_METADATA_PORT}", "hadoopConf":{"hive.metastore.execute.setugi":"true"}}', now(), NULL, now(), NULL); ``` -如果hive数据源在部署时设置了需要进行kerberos方式认证，则需要在linkis_ps_dm_datasource_env表的parameter字段指定一个参数keyTab，其值的获取方式可见：[在Linkis中设置并认证hive数据源](https://linkis.apache.org/zh-CN/docs/1.1.1/deployment/start-metadatasource) +如果hive数据源在部署时设置了需要进行kerberos方式认证，则需要在linkis_ps_dm_datasource_env表的parameter字段指定一个参数keyTab，其值的获取方式可见：[在Linkis中设置并认证hive数据源](https://linkis.apache.org/zh-CN/docs/latest/auth/token) #### 1.4 底层依赖组件检查 -**请确保 DSS1.1.0 与 Linkis1.1.1 基本可用，可在 DSS 前端界面执行 HiveQL 脚本，可正常创建并执行 DSS 工作流。** +**请确保 DSS1.1.2 与 Linkis1.4.0 基本可用，可在 DSS 前端界面执行 HiveQL 脚本，可正常创建并执行 DSS 工作流。** ## 2. Exchangis 安装部署 @@ -70,7 +71,7 @@ INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_t #### 2.1.1下载二进制包 -从 Exchangis 已发布的 release 中 [点击下载exchangis安装包](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.0.0)，下载最新的安装包。 +从 Exchangis 已发布的 release 中 [点击下载exchangis安装包](https://github.com/WeBankFinTech/Exchangis/releases/tag/release-1.1.2)，下载最新的安装包。 #### 2.1.2 编译打包 @@ -95,8 +96,9 @@ INSERT INTO `linkis_ps_dm_datasource_env` (`env_name`, `env_desc`, `datasource_t ```html |-- config：一键安装部署参数配置目录 |-- db：数据库表初始化 SQL 目录 -|-- exchangis-extds |-- packages：Exchangis 安装包目录 + |-- exchangis-extds：数据源扩展库 + |-- lib：库 |-- sbin：脚本存放目录 ``` @@ -156,7 +158,28 @@ DATABASE={dbName} 输入 `y` 初始化数据库表，输入 `n` 跳过数据库表初始化步骤。 -#### 2.5.3 启动服务 +#### 2.5.3 修改配置文件路径和日志文件路径 + +在sbin目录下的`env.properties`文件，设置配置文件路径和日志文件路径 + +```yaml +EXCHANGIS_CONF_PATH="/appcom/config/exchangis-config/background" +EXCHANGIS_LOG_PATH="/appcom/logs/exchangis/background" +MODULE_DEFAULT_PREFIX="dss-exchangis-main-" +MODULE_DEFAULT_SUFFIX="-dev" +``` + +EXCHANGIS_CONF_PATH为配置文件路径，EXCHANGIS_LOG_PATH为日志文件路径，若为以上配置，则作如下操作： + +```shell +cd {EXCHANGIS_DEPLOY_PATH} +cp -r config /appcom/config/exchangis-config/background +mkdir -p /appcom/logs/exchangis/background +``` + +则在服务启动时，将会使用对应路径下的配置文件，以及将日志写到对应的路径下 + +#### 2.5.4 启动服务 第一次启动，可以sbin目录下执行以下命令，启动 Exchangis Server： @@ -172,7 +195,7 @@ DATABASE={dbName} 执行完成启动脚本后，会出现以下提示，eureka地址也会在启动服务时在控制台打出： -![企业微信截图_16532930262583](https://user-images.githubusercontent.com/27387830/173892397-7cc7e988-0222-4f64-92ed-2cc58669770e.png) +![企业微信截图_16532930262583](../../../images/zh_CN/ch1/register_eureka.png) ### 2.6 查看服务是否启动成功 @@ -188,7 +211,7 @@ DATABASE={dbName} #### 2.7.1 获取前端安装包 -Exchangis 已默认提供了编译好的前端安装包，可直接下载使用：[点击下载前端安装包](https://github.com/WeBankFinTech/Exchangis/releases/download/release-1.0.0-rc1/web-dist.zip) +Exchangis 已默认提供了编译好的前端安装包，可直接下载使用：[点击下载前端安装包](https://github.com/WeBankFinTech/Exchangis/releases/download/release-1.1.2/web-dist.zip) 您也可以自行编译 Exchangis 前端，在 Exchangis 根目录下执行如下命令： @@ -198,23 +221,22 @@ Exchangis 已默认提供了编译好的前端安装包，可直接下载使用 npm run build ``` -从 `web/` 路径获取编译好的 dist.zip 前端包。 +从 `web/` 路径获取编译好的 exchangis-ui.zip 前端包。 获取到的前端包，您可以放在服务器上的任意位置，这里建议您与后端安装地址目录保持一致，在同一目录下放置并解压。 -#### 2.7.2 前端安装部署 +#### 3.3.4 前端安装部署 1. 解压前端安装包 -如您打算将 Exchangis 前端包部署到 `/appcom/Install/exchangis/web` 目录，请先将 `dist.zip` 拷贝到该目录并执行解压，注意，**请在安装dss的机器上安装exchangis前端**： +如您打算将 Exchangis 前端包部署到 `/appcom/Install/ExchangisInstall/exchangis-ui` 目录，请先将 `exchangis-ui.zip` 拷贝到该目录并执行解压，注意，**请在安装dss的机器上安装exchangis前端**： ```shell script - # 请先将 Exchangis 前端包拷贝到 `/appcom/Install/exchangis/web` 目录 - cd /appcom/Install/exchangis/web - unzip dist.zip + # 请先将 Exchangis 前端包拷贝到 `/appcom/Install/ExchangisInstall` 目录 + cd /appcom/Install/ExchangisInstall + unzip exchangis-ui.zip ``` - 执行如下命令： ```shell script @@ -227,8 +249,8 @@ Exchangis 已默认提供了编译好的前端安装包，可直接下载使用 server_name localhost; #charset koi8-r; #access_log /var/log/nginx/host.access.log main; - location /dist { - root /appcom/Install/exchangis/web; # Exchangis 前端部署目录 + location / { + root /appcom/Install/ExchangisInstall/exchangis-ui; # Exchangis 前端部署目录 autoindex on; } @@ -271,12 +293,16 @@ Exchangis 已默认提供了编译好的前端安装包，可直接下载使用 ## 3. DSS ExchangisAppConn 安装部署 -如您想正常使用 Exchangis1.0.0 前端，还需安装 DSS ExchangisAppConn 插件，请参考: [ExchangisAppConn 插件安装文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) +如您想正常使用 Exchangis1.1.2 前端，还需安装 DSS ExchangisAppConn 插件，请参考: [ExchangisAppConn 插件安装文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.1.2/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md) ## 4. Linkis Sqoop 引擎安装部署 -如您想正常执行 Exchangis1.0.0 的 Sqoop作业，还需安装 Linkis Sqoop 引擎，请参考: [Linkis Sqoop 引擎插件安装文档](https://linkis.staged.apache.org/zh-CN/docs/1.1.2/engine_usage/sqoop) +如您想正常执行 Exchangis1.1.2 的 Sqoop作业，还需安装 Linkis Sqoop 引擎，请参考: [Linkis Sqoop 引擎插件安装文档](https://linkis.staged.apache.org/zh-CN/docs/1.4.0/engine_usage/sqoop) + +## 5. Linkis DataX 引擎安装部署 + +如您想正常执行 Exchangis1.0.0 的 DataX作业，还需安装 Linkis DataX 引擎，请参考: [Linkis DataX 引擎插件安装文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.1.2/docs/zh_CN/ch1/exchangis_datax_deploy_cn.md) -## 5. 如何登录使用 Exchangis +## 6. 如何登录使用 Exchangis -Exchangis1.0更多使用说明，请参考用户使用手册[Exchangis1.0 用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_user_manual_cn.md) +Exchangis更多使用说明，请参考用户使用手册[Exchangis 用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.1.2/docs/zh_CN/ch1/exchangis_user_manual_cn.md) diff --git a/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md b/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md index 20eafb5f7..817854020 100644 --- a/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md @@ -37,12 +37,12 @@ Exchangis1.0.0和Linkis 1.1.1支持的主流Sqoop版本1.4.6与1.4.7，更高版 2.在exchangis-plugins模块下，找到sqoop引擎，单独编译sqoop，操作如下 ``` -cd {EXCHANGIS_CODE_HOME}/exchangis-plugins/engine/sqoop +cd {EXCHANGIS_CODE_HOME}/exchangis-engines/engine-plugins/sqoop mvn clean install ``` 然后会在该路径下找到sqoop引擎安装包 ``` -{EXCHANGIS_CODE_HOME}/exchangis-plugins/sqoop/target/out +{EXCHANGIS_CODE_HOME}/exchangis-engines/engine-plugins/sqoop/target/out ``` @@ -59,9 +59,9 @@ sqoop 2.放置到linkis安装路径的如下目录 ```shell -cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins +cd {LINKIS_HOME}/linkis-engineconn-plugins ``` -(注意，看当前sqoop引擎对哪些用户有权限，不一定是root) +（注意，看当前sqoop引擎对哪些用户有权限，不一定是root） #### 2）重启linkis-engineplugin服务使sqoop引擎生效 @@ -70,7 +70,13 @@ cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins cd {LINKIS_INSTALL_HOME}/links/sbin/ ./linkis-daemon.sh restart cg-engineplugin ``` -待服务启动成功，至此，sqoop安装部署就完成了。 +待服务启动成功，在linkis数据库中校验sqoop引擎是否安装完毕 + +```yaml +select * from linkis_cg_engine_conn_plugin_bml_resources where engine_conn_type='sqoop'; +``` + +至此，sqoop安装部署就完成了。 engineplugin更详细的介绍可以参看下面的文章。 -https://linkis.apache.org/zh-CN/docs/latest/deployment/engine-conn-plugin-installation/ +https://linkis.apache.org/zh-CN/docs/latest/deployment/engine-conn-plugin-installation/ \ No newline at end of file diff --git a/exchangis-dao/pom.xml b/exchangis-dao/pom.xml index 43714347a..5776a5b52 100644 --- a/exchangis-dao/pom.xml +++ b/exchangis-dao/pom.xml @@ -5,7 +5,7 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -21,24 +21,20 @@ org.apache.linkis linkis-mybatis - ${linkis.version} org.apache.linkis linkis-module - ${linkis.version} - - - validation-api - javax.validation - - org.hibernate hibernate-validator ${hibernate.validator} + + org.springframework + spring-orm + diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/AuditLogUtils.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/AuditLogUtils.java index 40a09abed..f42908668 100644 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/AuditLogUtils.java +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/AuditLogUtils.java @@ -31,7 +31,7 @@ public static void printLog(String user, String proxyUser, TargetTypeEnum target //String detailInfo=new Gson().toJson(params); String detailInfo=params.toString(); LOGGER.info("[{}],[{}],[{}],[{}],[{}],[{}],[{}],[{}],[{}]", - new Date(),user, "proxyUser is: " + proxyUser, "Exchangis-1.1.1", targetType.getName(), + new Date(),user, "proxyUser is: " + proxyUser, "Exchangis-1.1.2", targetType.getName(), targetId,targetName,operateType.getName(), detailInfo); } diff --git a/exchangis-datasource/exchangis-datasource-core/pom.xml b/exchangis-datasource/exchangis-datasource-core/pom.xml index ecca5a44f..990c92d6f 100644 --- a/exchangis-datasource/exchangis-datasource-core/pom.xml +++ b/exchangis-datasource/exchangis-datasource-core/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -20,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java index 60400f70f..f68f696e8 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java @@ -10,7 +10,6 @@ public interface ExchangisDataSourceLoader { -// String EXCHANGIS_DIR_NAME = "exchangis-extds"; String EXCHANGIS_DIR_NAME = Objects.isNull(CommonVars.apply("wds.exchangis.datasource.extension.dir").getValue()) ? "exchangis-extds" : CommonVars.apply("wds.exchangis.datasource.extension.dir").getValue().toString(); String PROPERTIES_NAME = "extds.properties"; diff --git a/exchangis-datasource/exchangis-datasource-linkis/pom.xml b/exchangis-datasource/exchangis-datasource-linkis/pom.xml index 4acb1d9b8..2ab7eda3d 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/pom.xml +++ b/exchangis-datasource/exchangis-datasource-linkis/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -20,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java index 4237037b5..5bbe000e2 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java +++ b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java @@ -3,11 +3,9 @@ import com.webank.wedatasphere.exchangis.common.EnvironmentUtils; import com.webank.wedatasphere.exchangis.datasource.core.domain.MetaColumn; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; -import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisServiceRpcException; import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; import com.webank.wedatasphere.exchangis.datasource.core.service.rpc.ServiceRpcClient; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisLinkisRemoteClient; -//import com.webank.wedatasphere.exchangis.datasource.linkis.partition.MetadataGetPartitionsResult; import com.webank.wedatasphere.exchangis.datasource.linkis.request.MetadataGetConnInfoAction; import com.webank.wedatasphere.exchangis.datasource.linkis.request.MetadataGetPartitionPropsAction; import com.webank.wedatasphere.exchangis.datasource.linkis.response.MetadataGetConnInfoResult; @@ -21,7 +19,7 @@ import org.apache.linkis.datasource.client.response.MetadataGetColumnsResult; import org.apache.linkis.datasource.client.response.MetadataGetPartitionsResult; import org.apache.linkis.datasource.client.response.MetadataGetTablePropsResult; -import org.apache.linkis.metadatamanager.common.domain.MetaColumnInfo; +import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import java.util.*; diff --git a/exchangis-datasource/exchangis-datasource-loader/pom.xml b/exchangis-datasource/exchangis-datasource-loader/pom.xml index 91978eec3..30ebb07ad 100644 --- a/exchangis-datasource/exchangis-datasource-loader/pom.xml +++ b/exchangis-datasource/exchangis-datasource-loader/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -20,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java index b29873bf9..50f6f59e3 100644 --- a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java +++ b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java @@ -7,6 +7,7 @@ import com.webank.wedatasphere.exchangis.datasource.loader.clazzloader.ExchangisDataSourceClassLoader; import com.webank.wedatasphere.exchangis.datasource.loader.utils.ExceptionHelper; import com.webank.wedatasphere.exchangis.datasource.loader.utils.ExtDsUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.linkis.common.exception.ErrorException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,7 +39,9 @@ public void init(MapperHook mapperHook) throws Exception { // 初始化磁盘扫描加载 ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); String loadClassPath = Objects.requireNonNull(currentClassLoader.getResource(EXCHANGIS_DIR_NAME)).getPath(); -// String libPathUrl = loadClassPath + ".." + File.separator + ".." + File.separator + EXCHANGIS_DIR_NAME; + if (StringUtils.endsWith(loadClassPath, File.separator)) { + loadClassPath = loadClassPath + File.separator; + } String libPathUrl = loadClassPath + ".." + File.separator + EXCHANGIS_DIR_NAME; LOGGER.info("libPath url is {}", libPathUrl); List jars = ExtDsUtils.getJarsUrlsOfPath(libPathUrl); diff --git a/exchangis-datasource/exchangis-datasource-server/pom.xml b/exchangis-datasource/exchangis-datasource-server/pom.xml index f759adf04..c41abaa25 100644 --- a/exchangis-datasource/exchangis-datasource-server/pom.xml +++ b/exchangis-datasource/exchangis-datasource-server/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -20,13 +20,13 @@ com.webank.wedatasphere.exchangis exchangis-datasource-service - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-loader - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/exchangis-datasource-service/pom.xml b/exchangis-datasource/exchangis-datasource-service/pom.xml index 157270d7b..e41e65f03 100644 --- a/exchangis-datasource/exchangis-datasource-service/pom.xml +++ b/exchangis-datasource/exchangis-datasource-service/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -21,19 +21,19 @@ com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-engine-core - 1.1.1 + 1.1.2 compile diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java index 9527aff11..40fcced7b 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java @@ -43,7 +43,7 @@ import org.apache.linkis.datasourcemanager.common.exception.JsonErrorException; import org.apache.linkis.datasourcemanager.common.util.json.Json; import org.apache.linkis.httpclient.response.Result; -import org.apache.linkis.metadatamanager.common.domain.MetaColumnInfo; +import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import org.apache.linkis.server.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -401,7 +401,7 @@ public Message deleteDataSource(HttpServletRequest request, /*String type,*/ Lon QueryWrapper condition = new QueryWrapper<>(); condition.eq("source_ds_id", id).or().eq("sink_ds_id", id); - Integer inUseCount = this.exchangisJobDsBindMapper.selectCount(condition); + Long inUseCount = this.exchangisJobDsBindMapper.selectCount(condition); if (inUseCount > 0) { throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_DATASOURCE_DELETE_ERROR.getCode(), "目前存在引用依赖"); } @@ -630,7 +630,7 @@ public Message queryDataSources(HttpServletRequest request, DataSourceQueryVO vo Integer page = Objects.isNull(vo.getPage()) ? 1 : vo.getPage(); Integer pageSize = Objects.isNull(vo.getPageSize()) ? 100 : vo.getPageSize(); - String dataSourceName = Objects.isNull(vo.getName()) ? "" : vo.getName(); + String dataSourceName = Objects.isNull(vo.getName()) ? "" : vo.getName().replace("_", "\\_"); LinkisDataSourceRemoteClient linkisDataSourceRemoteClient = ExchangisLinkisRemoteClient.getLinkisDataSourceRemoteClient(); QueryDataSourceResult result; int totalPage = 0; diff --git a/exchangis-datasource/exchangis-datasource-streamis/pom.xml b/exchangis-datasource/exchangis-datasource-streamis/pom.xml index a3695e549..906e781a2 100644 --- a/exchangis-datasource/exchangis-datasource-streamis/pom.xml +++ b/exchangis-datasource/exchangis-datasource-streamis/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -20,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml index 6ea20452a..a92fa201c 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml index ebc851eb6..c1e258f85 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -22,17 +22,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml index 60f6b2afd..a26312d6e 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml index 6201fc9e2..71bb30072 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml index 7d6288c5b..e3cce71da 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml index d02a5c64d..47af538ab 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.1 + 1.1.2 diff --git a/exchangis-datasource/pom.xml b/exchangis-datasource/pom.xml index ce24d510b..ad1bd9fae 100644 --- a/exchangis-datasource/pom.xml +++ b/exchangis-datasource/pom.xml @@ -5,13 +5,12 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 exchangis-datasource pom - 1.1.1 exchangis-datasource-core diff --git a/exchangis-engines/engineconn-plugins/datax/pom.xml b/exchangis-engines/engineconn-plugins/datax/pom.xml index 1da0bb390..dd10e755f 100644 --- a/exchangis-engines/engineconn-plugins/datax/pom.xml +++ b/exchangis-engines/engineconn-plugins/datax/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -13,10 +13,10 @@ linkis-engineplugin-datax - 1.1.6-webank + 1.4.0 3.0.0-Plus-2 3.0.0 - 2.7.2 + 3.3.4 1.0.15 2.1.9 0.3 @@ -90,7 +90,6 @@ org.apache.linkis linkis-common ${linkis.version} - provided commons-cli @@ -102,6 +101,21 @@ commons-configuration ${commons-configuration-version} + + org.apache.linkis + linkis-computation-engineconn + 1.4.0 + + + org.apache.linkis + linkis-udf-common + 1.4.0 + + + org.apache.linkis + linkis-udf-client + 1.4.0 + @@ -176,4 +190,15 @@ + + + + apache.snapshots + Apache Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + true + + + \ No newline at end of file diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml b/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml index 667d0f544..6c517ef5b 100644 --- a/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml +++ b/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml @@ -33,7 +33,7 @@ - /dist/v${datax.version}/lib + /dist/${datax.version}/lib true true false @@ -294,7 +294,6 @@ - ${basedir}/src/main/resources @@ -302,10 +301,9 @@ 0777 0755 - /dist/v${datax.version}/conf + /dist/${datax.version}/conf unix - ${basedir}/target @@ -317,9 +315,7 @@ 0777 /plugin/${datax.version} - - diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala index 34f6c4e4a..996e4da90 100644 --- a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala +++ b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/DataxEngineConnPlugin.scala @@ -26,7 +26,8 @@ import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuil import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceFactory import org.apache.linkis.manager.label.entity.Label -import java.util +import java.util.List +import java.util.ArrayList class DataxEngineConnPlugin extends EngineConnPlugin { @@ -36,7 +37,7 @@ class DataxEngineConnPlugin extends EngineConnPlugin { private var engineConnFactory: EngineConnFactory = _ private val engineConnFactoryLocker = new Array[Byte](0) - override def init(params: util.Map[String, Any]): Unit = {} + override def init(params: java.util.Map[_root_.scala.Predef.String, scala.AnyRef]): Unit = {} override def getEngineResourceFactory: EngineResourceFactory = { if (null == engineResourceFactory) engineResourceFactoryLocker.synchronized { @@ -60,7 +61,7 @@ class DataxEngineConnPlugin extends EngineConnPlugin { engineConnFactory } - override def getDefaultLabels: util.List[Label[_]] = { - new util.ArrayList[Label[_]]() + override def getDefaultLabels: List[Label[_]] = { + new ArrayList[Label[_]]() } } diff --git a/exchangis-engines/engineconn-plugins/sqoop/pom.xml b/exchangis-engines/engineconn-plugins/sqoop/pom.xml index 65aaab13b..ee8398bf7 100644 --- a/exchangis-engines/engineconn-plugins/sqoop/pom.xml +++ b/exchangis-engines/engineconn-plugins/sqoop/pom.xml @@ -22,7 +22,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../../pom.xml 4.0.0 @@ -30,9 +30,9 @@ linkis-engineplugin-sqoop 1.4.6 - 3.1.2 - 2.7.2 - 1.1.6-webank + 3.1.3 + 3.3.4 + 1.4.0 @@ -180,6 +180,10 @@ hadoop-hdfs org.apache.hadoop + + org.apache.hadoop + hadoop-hdfs + @@ -187,28 +191,24 @@ linkis-engineconn-plugin-core ${linkis.version} - org.apache.linkis linkis-rpc ${linkis.version} provided - org.apache.linkis linkis-storage ${linkis.version} provided - org.apache.linkis linkis-common ${linkis.version} provided - @@ -217,7 +217,6 @@ org.apache.maven.plugins maven-deploy-plugin - net.alchim31.maven scala-maven-plugin diff --git a/exchangis-engines/engineconn-plugins/sqoop/src/main/assembly/distribution.xml b/exchangis-engines/engineconn-plugins/sqoop/src/main/assembly/distribution.xml index 78f54c4c0..a88001e18 100644 --- a/exchangis-engines/engineconn-plugins/sqoop/src/main/assembly/distribution.xml +++ b/exchangis-engines/engineconn-plugins/sqoop/src/main/assembly/distribution.xml @@ -32,7 +32,7 @@ - /dist/v${sqoop.version}/lib + /dist/${sqoop.version}/lib true true false diff --git a/exchangis-engines/engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/SqoopEngineConnPlugin.scala b/exchangis-engines/engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/SqoopEngineConnPlugin.scala index 52266bb71..ae322a3f5 100644 --- a/exchangis-engines/engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/SqoopEngineConnPlugin.scala +++ b/exchangis-engines/engineconn-plugins/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/SqoopEngineConnPlugin.scala @@ -25,16 +25,17 @@ import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.engineconnplugin.sqoop.factory.SqoopEngineConnFactory import org.apache.linkis.engineconnplugin.sqoop.launch.SqoopEngineConnLaunchBuilder +import java.util.Map +import java.util.List class SqoopEngineConnPlugin extends EngineConnPlugin{ private val EP_CONTEXT_CONSTRUCTOR_LOCK = new Object() private var engineResourceFactory: EngineResourceFactory = _ private var engineConnLaunchBuilder: EngineConnLaunchBuilder = _ private var engineConnFactory: EngineConnFactory = _ - override def init(params: java.util.Map[String, Any]): Unit = {} + override def init(params: Map[String, AnyRef]): Unit = {} override def getEngineResourceFactory: EngineResourceFactory = { - EP_CONTEXT_CONSTRUCTOR_LOCK.synchronized{ if(null == engineResourceFactory){ engineResourceFactory = new GenericEngineResourceFactory @@ -62,5 +63,6 @@ class SqoopEngineConnPlugin extends EngineConnPlugin{ } } - override def getDefaultLabels: java.util.List[Label[_]] = new java.util.ArrayList[Label[_]] + override def getDefaultLabels: List[Label[_]] = new java.util.ArrayList[Label[_]] + } diff --git a/exchangis-engines/engines/datax/datax-assembly/pom.xml b/exchangis-engines/engines/datax/datax-assembly/pom.xml index 560ecbc90..74b755512 100644 --- a/exchangis-engines/engines/datax/datax-assembly/pom.xml +++ b/exchangis-engines/engines/datax/datax-assembly/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-core/pom.xml b/exchangis-engines/engines/datax/datax-core/pom.xml index 78134c869..48be143d3 100644 --- a/exchangis-engines/engines/datax/datax-core/pom.xml +++ b/exchangis-engines/engines/datax/datax-core/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 @@ -79,6 +79,7 @@ ch.qos.logback logback-classic + provided org.slf4j diff --git a/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml b/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml index e844cdf77..357a9b855 100644 --- a/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-ftpreader/pom.xml b/exchangis-engines/engines/datax/datax-ftpreader/pom.xml index 6e1a4d70a..8520f0145 100644 --- a/exchangis-engines/engines/datax/datax-ftpreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-ftpreader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml b/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml index 37a55d81f..ec0635c29 100644 --- a/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml index 220ed8051..81789b25b 100644 --- a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 @@ -13,13 +13,12 @@ jar 3.0.0-Plus-2 - 1.2.1 - 2.7.2 - 1.2.2 + 3.1.3 + 3.3.4 + 2.5.5 2.9.1 - com.webank.wedatasphere.exchangis datax-core @@ -61,7 +60,7 @@ io.netty netty-all - 4.1.46.Final + 4.1.86.Final org.apache.hadoop @@ -108,6 +107,10 @@ hbase-server ${hbase.version} + + org.apache.hadoop + hadoop-hdfs + diff --git a/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/hfile/HBASEV1HFileParser.java b/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/hfile/HBASEV1HFileParser.java index 0fe8d4a69..b46189e7c 100644 --- a/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/hfile/HBASEV1HFileParser.java +++ b/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/hfile/HBASEV1HFileParser.java @@ -47,19 +47,18 @@ public void parse(String inputPath, Configuration parseConf, Action action) { org.apache.hadoop.conf.Configuration configuration = fileSystem.getConf(); LOG.info("Start to parse HFile: [" + inputPath + "] in HBASEV1HFileParser"); try (HFile.Reader reader = HFile.createReader(fileSystem, new Path(inputPath), - new CacheConfig(configuration), configuration)) { - HFileScanner scanner = reader.getScanner(true, true); + new CacheConfig(configuration), false, configuration)) { + HFileScanner scanner = reader.getScanner(configuration, true, true); if(null == parseConf){ parseConf = Configuration.from("{}"); for(String parseColumn : PARSE_COLUMNS){ parseConf.set(parseColumn, true); } } - reader.loadFileInfo(); if(scanner.seekTo()) { do { //Cell entity - Cell cell = scanner.getKeyValue(); + Cell cell = scanner.getCell(); List sourceList = new ArrayList<>(); parseConf.getKeys().forEach(configKey -> { switch(configKey){ diff --git a/exchangis-engines/engines/datax/datax-hdfsreader/src/test/java/com/alibaba/datax/plugin/reader/hdfsreader/HFileParser.java b/exchangis-engines/engines/datax/datax-hdfsreader/src/test/java/com/alibaba/datax/plugin/reader/hdfsreader/HFileParser.java index d40a9644c..dae334eab 100644 --- a/exchangis-engines/engines/datax/datax-hdfsreader/src/test/java/com/alibaba/datax/plugin/reader/hdfsreader/HFileParser.java +++ b/exchangis-engines/engines/datax/datax-hdfsreader/src/test/java/com/alibaba/datax/plugin/reader/hdfsreader/HFileParser.java @@ -4,7 +4,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; @@ -15,11 +14,10 @@ public class HFileParser { public void parse(String pathString, Configuration hadoopConf) throws IOException { FileSystem fs = new Path(pathString).getFileSystem(hadoopConf); - HFile.Reader reader = HFile.createReader(fs, new Path(pathString), new CacheConfig(hadoopConf), hadoopConf); - HFileScanner scanner = reader.getScanner(true, true); - reader.loadFileInfo(); + HFile.Reader reader = HFile.createReader(fs, new Path(pathString), new CacheConfig(hadoopConf), false, hadoopConf); + HFileScanner scanner = reader.getScanner(hadoopConf, true, true); scanner.seekTo(); - Cell cell = scanner.getKeyValue(); + Cell cell = scanner.getCell(); scanner.next(); } } diff --git a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml index e186cb375..11c814aec 100644 --- a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 @@ -14,8 +14,8 @@ jar 3.0.0-Plus-2 - 1.2.1 - 2.7.2 + 3.1.3 + 3.3.4 2.9.1 @@ -46,7 +46,6 @@ logback-classic provided - org.apache.hadoop hadoop-hdfs @@ -66,7 +65,7 @@ io.netty netty-all - 4.1.46.Final + 4.1.86.Final org.apache.hadoop diff --git a/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml b/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml index a72c46afd..12389b823 100644 --- a/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml b/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml index dbf27f6de..7339dbc27 100644 --- a/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-oraclereader/pom.xml b/exchangis-engines/engines/datax/datax-oraclereader/pom.xml index 9754669a9..fd6f86f07 100644 --- a/exchangis-engines/engines/datax/datax-oraclereader/pom.xml +++ b/exchangis-engines/engines/datax/datax-oraclereader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml b/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml index 82db85b0c..054951019 100644 --- a/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-textfilereader/pom.xml b/exchangis-engines/engines/datax/datax-textfilereader/pom.xml index 452119ba6..6e7c18313 100644 --- a/exchangis-engines/engines/datax/datax-textfilereader/pom.xml +++ b/exchangis-engines/engines/datax/datax-textfilereader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml b/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml index 6e1a746bd..1d0c9f1f3 100644 --- a/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/pom.xml b/exchangis-engines/engines/datax/pom.xml index ff2abca29..15d7f5879 100644 --- a/exchangis-engines/engines/datax/pom.xml +++ b/exchangis-engines/engines/datax/pom.xml @@ -5,7 +5,7 @@ com.webank.wedatasphere.exchangis exchangis - 1.1.1 + 1.1.2 ../../../pom.xml @@ -25,8 +25,8 @@ 1.10 1.2 1.9.4 - 2.7.2 - 1.2.1 + 3.3.4 + 3.1.3 6.7.1 1.11 3.0.0-Plus-2 diff --git a/exchangis-engines/exchangis-engine-common/pom.xml b/exchangis-engines/exchangis-engine-common/pom.xml index 1501f3206..d6cf14efa 100644 --- a/exchangis-engines/exchangis-engine-common/pom.xml +++ b/exchangis-engines/exchangis-engine-common/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 diff --git a/exchangis-engines/exchangis-engine-core/pom.xml b/exchangis-engines/exchangis-engine-core/pom.xml index a551b75e6..5d040b341 100644 --- a/exchangis-engines/exchangis-engine-core/pom.xml +++ b/exchangis-engines/exchangis-engine-core/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 diff --git a/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/EngineResourceLoader.java b/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/EngineResourceLoader.java index ca948ae98..dd5a74241 100644 --- a/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/EngineResourceLoader.java +++ b/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/EngineResourceLoader.java @@ -17,14 +17,14 @@ public interface EngineResourceLoader { String engineType(); /** * Accept uri - * @param uri uri + * @param baseUri uri * @return boolean */ boolean accept(URI baseUri, String path); /** * Load resources from uri - * @param uri uri + * @param baseUri uri * @return resource array */ T[] loadResource(URI baseUri, String path) throws ExchangisEngineResLoadException; diff --git a/exchangis-engines/exchangis-engine-server/pom.xml b/exchangis-engines/exchangis-engine-server/pom.xml index fb7370bb6..5873fe538 100644 --- a/exchangis-engines/exchangis-engine-server/pom.xml +++ b/exchangis-engines/exchangis-engine-server/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 diff --git a/exchangis-engines/exchangis-engine-server/src/main/java/com/webank/wedatasphere/exchangis/engine/server/mapper/EngineResourceMapper.xml b/exchangis-engines/exchangis-engine-server/src/main/java/com/webank/wedatasphere/exchangis/engine/server/mapper/ProjectExportService.xml similarity index 100% rename from exchangis-engines/exchangis-engine-server/src/main/java/com/webank/wedatasphere/exchangis/engine/server/mapper/EngineResourceMapper.xml rename to exchangis-engines/exchangis-engine-server/src/main/java/com/webank/wedatasphere/exchangis/engine/server/mapper/ProjectExportService.xml diff --git a/exchangis-engines/pom.xml b/exchangis-engines/pom.xml index 783dbaac0..7dfac3c51 100644 --- a/exchangis-engines/pom.xml +++ b/exchangis-engines/pom.xml @@ -5,13 +5,13 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 exchangis-engines pom - 1.1.1 + 1.1.2 exchangis-engine-common diff --git a/exchangis-job/exchangis-job-builder/pom.xml b/exchangis-job/exchangis-job-builder/pom.xml index 18e6e36e1..acfbb460a 100644 --- a/exchangis-job/exchangis-job-builder/pom.xml +++ b/exchangis-job/exchangis-job-builder/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -15,7 +15,7 @@ com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.1 + 1.1.2 com.google.code.gson @@ -25,7 +25,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-service - 1.1.1 + 1.1.2 compile diff --git a/exchangis-job/exchangis-job-common/pom.xml b/exchangis-job/exchangis-job-common/pom.xml index a15fc587c..1bae2fa82 100644 --- a/exchangis-job/exchangis-job-common/pom.xml +++ b/exchangis-job/exchangis-job-common/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -15,7 +15,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.1 + 1.1.2 org.apache.linkis @@ -25,7 +25,7 @@ com.webank.wedatasphere.exchangis exchangis-engine-common - 1.1.1 + 1.1.2 org.apache.linkis diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobQueryVo.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobQueryVo.java index d0ba5d4f9..3806e8c44 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobQueryVo.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobQueryVo.java @@ -4,8 +4,15 @@ public class ExchangisJobQueryVo extends ExchangisJobPageQuery { + private static final Integer defaultCurrentPage = 1; + + private static final Integer defaultPageSize = 10; + public ExchangisJobQueryVo(){ + } + public ExchangisJobQueryVo(Long projectId, String jobType, String name) { + this(projectId, jobType, name, defaultCurrentPage, defaultPageSize); } public ExchangisJobQueryVo(Long projectId, String jobType, diff --git a/exchangis-job/exchangis-job-launcher/pom.xml b/exchangis-job/exchangis-job-launcher/pom.xml index 88aabb8d6..2375cdea3 100644 --- a/exchangis-job/exchangis-job-launcher/pom.xml +++ b/exchangis-job/exchangis-job-launcher/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -15,12 +15,12 @@ com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.1 + 1.1.2 com.webank.wedatasphere.exchangis exchangis-job-builder - 1.1.1 + 1.1.2 org.apache.linkis diff --git a/exchangis-job/exchangis-job-metrics/pom.xml b/exchangis-job/exchangis-job-metrics/pom.xml index 52b3f3f73..9dcdd12fb 100644 --- a/exchangis-job/exchangis-job-metrics/pom.xml +++ b/exchangis-job/exchangis-job-metrics/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 diff --git a/exchangis-job/exchangis-job-server/pom.xml b/exchangis-job/exchangis-job-server/pom.xml index 3a370b347..60c64165e 100644 --- a/exchangis-job/exchangis-job-server/pom.xml +++ b/exchangis-job/exchangis-job-server/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 @@ -17,24 +17,21 @@ - com.webank.wedatasphere.exchangis - exchangis-job-common - 1.1.1 + exchangis-project-provider + 1.1.2 - com.webank.wedatasphere.exchangis - exchangis-datasource-service - 1.1.1 + exchangis-job-launcher + 1.1.2 com.webank.wedatasphere.exchangis - exchangis-job-launcher - 1.1.1 + exchangis-datasource-service + 1.1.2 - com.webank.wedatasphere.exchangis exchangis-engine-core @@ -52,12 +49,6 @@ mysql-connector-java 5.1.49 - - com.webank.wedatasphere.exchangis - exchangis-project-server - 1.1.1 - compile - diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java index 8d714fdb5..031425be7 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java @@ -199,7 +199,7 @@ private enum Type { } path = pathBuilder.toString(); } - return path; + return path.replaceAll(" ", "%20"); }); /** diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java index 1adc16108..6cbbc64a9 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java @@ -17,11 +17,9 @@ import com.webank.wedatasphere.exchangis.job.server.render.transform.field.mapping.match.FieldMatchStrategy; import com.webank.wedatasphere.exchangis.job.server.utils.SpringContextHolder; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java index 94fcf6636..73929af81 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java @@ -17,8 +17,6 @@ import com.webank.wedatasphere.exchangis.job.server.vo.JobFunction; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; -import com.webank.wedatasphere.exchangis.project.server.exception.ExchangisProjectErrorException; -import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.ProxyUserSSOUtils; import org.apache.linkis.server.security.SecurityFilter; @@ -36,7 +34,6 @@ import java.util.List; import java.util.Objects; - /** * The basic controller of Exchangis job */ @@ -52,9 +49,6 @@ public class ExchangisJobRestfulApi { @Resource private JobInfoService jobInfoService; - @Resource - private ProjectService projectService; - @Resource private JobFuncService jobFuncService; @@ -170,7 +164,7 @@ public Message jobFuncList(@PathVariable("tabName") String tabName, public Message createJob( @Validated({InsertGroup.class, Default.class}) @RequestBody ExchangisJobVo exchangisJobVo, BindingResult result, - HttpServletRequest request) throws ExchangisProjectErrorException { + HttpServletRequest request) { if (ExchangisLauncherConfiguration.LIMIT_INTERFACE.getValue()) { return Message.error("You have no permission to create Job (没有创建任务权限)"); } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java index 4e1b84886..5ed4cfb4a 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java @@ -218,6 +218,7 @@ public Message listJobs(@RequestParam(value = "jobExecutionId", required = false @RequestParam(value = "size", required = false) int size, HttpServletRequest request) { Message message = Message.ok("Submitted succeed(提交成功)！"); + jobName = jobName.replace("_", "\\_"); try { List jobList = executeService.getExecutedJobList(jobExecutionId, jobName, status, launchStartTime, launchEndTime, current, size, request); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java index 0ea358388..9b91a5814 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java @@ -9,22 +9,16 @@ import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; import com.webank.wedatasphere.exchangis.job.domain.OperationType; import com.webank.wedatasphere.exchangis.job.launcher.ExchangisLauncherConfiguration; -import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.job.server.service.IProjectCopyService; import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; import com.webank.wedatasphere.exchangis.job.server.service.impl.DefaultJobExecuteService; -import com.webank.wedatasphere.exchangis.job.server.service.impl.ProjectImportServerImpl; import com.webank.wedatasphere.exchangis.job.server.utils.JobAuthorityUtils; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; -import com.webank.wedatasphere.exchangis.project.server.mapper.ProjectMapper; import org.apache.commons.lang.StringUtils; import org.apache.linkis.server.BDPJettyServerHelper; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.validation.BindingResult; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; @@ -32,8 +26,6 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.validation.groups.Default; -import javax.ws.rs.core.Context; -import java.rmi.ServerException; import java.util.Map; import java.util.Objects; @@ -57,15 +49,6 @@ public class ExchangisJobDssAppConnRestfulApi { @Resource private DefaultJobExecuteService executeService; - @Resource - private ProjectImportServerImpl projectImportServer; - - @Resource - private IProjectCopyService projectCopyService; - - @Autowired - private ProjectMapper projectMapper; - /** * Create job * @param request http request @@ -213,9 +196,6 @@ public Message executeJob(@PathVariable("id") Long id, HttpServletRequest reques jobInfo.setId(jobVo.getId()); LOG.info("jobInfo: name{},executerUser{},createUser{},id{}",jobInfo.getName(),jobInfo.getExecuteUser(),jobInfo.getCreateUser(),jobInfo.getId()); LOG.info("loginUser: {}, jobVo:{}",loginUser,jobVo); - //find project info - ExchangisProject project = projectMapper.getDetailById(jobVo.getProjectId()); - LOG.info("project: {}, getProjectId:{}",project,jobVo.getProjectId()); //find project user authority /*if (!hasAuthority(submitUser, jobVo)){ return Message.error("You have no permission to execute job (没有执行DSS任务权限)"); @@ -254,71 +234,4 @@ public Message executeJob(@PathVariable("id") Long id, HttpServletRequest reques AuditLogUtils.printLog(oringinUser, loginUser, TargetTypeEnum.JOB, id.toString(), "Execute task is: " + jobInfo.getName(), OperateTypeEnum.EXECUTE, request); return result; } - - @RequestMapping( value = "/import", method = RequestMethod.POST) - public Message importJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException{ - - Message response = null; - String userName = UserUtils.getLoginUser(request); - String oringinUser = SecurityFilter.getLoginUsername(request); - try { - LOG.info("param: {}", params); - /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { - return Message.error("You have no permission to import (没有导入权限)"); - }*/ - response = projectImportServer.importProject(request, params); - LOG.info("import job success"); - } catch (ExchangisJobServerException e){ - String message = "Fail import job [ id: " + params + "] (导入任务失败)"; - LOG.error(message, e); - response = Message.error(message); - } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.IMPORT, request); - return response; - - } - - @RequestMapping( value = "/export", method = RequestMethod.POST) - public Message exportJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { - String userName = UserUtils.getLoginUser(request); - - LOG.info("export function params: {}", params); - String oringinUser = SecurityFilter.getLoginUsername(request); - Message response = null; - try { - /*if (!hasAuthority(userName, jobInfoService.getJob(((Integer) params.get("sqoopIds")).longValue(), true))) { - return Message.error("You have no permission to export (没有导出权限)"); - }*/ - response = jobInfoService.exportProject(params, userName, request); - LOG.info("export job success"); - } catch (Exception e){ - String message = "Fail Export job [ id: " + params + "] (导出任务失败)"; - LOG.error(message, e); - response = Message.error(message); - } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Export parameter is: " + params.toString(), OperateTypeEnum.EXPORT, request); - return response; - } - - @RequestMapping( value = "/copy", method = RequestMethod.POST) - public Message copy(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException { - String userName = UserUtils.getLoginUser(request); - String oringinUser = SecurityFilter.getLoginUsername(request); - - LOG.info("copy function params: {}", params); - Message response = null; - try { - response = projectCopyService.copy(params, userName, request); - LOG.info("copy node success"); - } catch (Exception e){ - String message = "Fail Copy project [ id: " + params + "] (导出任务失败)"; - LOG.error(message, e); - response = Message.error(message); - } - AuditLogUtils.printLog(oringinUser, userName, TargetTypeEnum.JOB, "", "Copy parameter is: " + params.toString(), OperateTypeEnum.COPY, request); - return response; - - //return jobInfoService.exportProject(params, userName, request); - - } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectExportService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectExportService.java deleted file mode 100644 index feea880c0..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectExportService.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.service; - -import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; - -import javax.servlet.http.HttpServletRequest; -import java.util.Map; -import java.util.Set; - -/** - * @author tikazhang - * @Date 2022/3/15 9:30 - */ -public interface IProjectExportService { - String exportProject(Map params, String username, HttpServletRequest request) throws Exception; - - ExportedProject export(Long projectId, Map> moduleIdsMap, boolean partial); - - Map> getModuleIdsMap(Map params); - - Long getProjectId(Map> moduleIdsMap); -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java index 748d9e0bc..aee170717 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java @@ -2,21 +2,14 @@ import com.webank.wedatasphere.exchangis.common.pager.PageResult; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; -import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import org.apache.linkis.server.Message; import javax.servlet.http.HttpServletRequest; import java.util.List; -import java.rmi.ServerException; -import java.util.Map; -import java.util.Set; - - /** * The interface Exchangis job service. * @@ -106,18 +99,6 @@ public interface JobInfoService { */ ExchangisJobVo updateJobContent(ExchangisJobVo jobVo) throws ExchangisJobServerException, ExchangisDataSourceException; - /** - * Export exchangis job to BML. - * - * @param username params - * @return - */ - Message exportProject(Map params, String username, HttpServletRequest request) throws ExchangisJobServerException, ServerException; - - ExportedProject export(Long projectId, Map> moduleIdsMap, boolean partial, HttpServletRequest request) throws ExchangisJobServerException; - - Map> getModuleIdsMap(Map params); - /** * Copy job exchangis job basic info. * diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java index f2114c340..3acfbf5e1 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java @@ -6,8 +6,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import com.webank.wedatasphere.exchangis.common.pager.PageResult; import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobDsBind; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; @@ -16,24 +14,14 @@ import com.webank.wedatasphere.exchangis.datasource.core.vo.ExchangisJobInfoContent; import com.webank.wedatasphere.exchangis.datasource.service.ExchangisDataSourceService; import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; -import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; import com.webank.wedatasphere.exchangis.job.server.mapper.ExchangisJobEntityDao; -import com.webank.wedatasphere.exchangis.job.server.restful.external.ModuleEnum; import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; -import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; -import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.apache.linkis.bml.client.BmlClient; -import org.apache.linkis.bml.client.BmlClientFactory; -import org.apache.linkis.bml.protocol.BmlUploadResponse; import org.apache.linkis.common.utils.JsonUtils; import org.apache.linkis.manager.label.utils.LabelUtils; -import org.apache.linkis.server.BDPJettyServerHelper; -import org.apache.linkis.server.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -42,9 +30,6 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; -import java.rmi.ServerException; import java.util.*; import java.util.stream.Collectors; @@ -56,12 +41,6 @@ public class DefaultJobInfoService implements JobInfoService { private static final Logger LOG = LoggerFactory.getLogger(DefaultJobInfoService.class); - /** - * Project service - */ - @Resource - private ProjectService projectService; - @Autowired private ExchangisJobDsBindServiceImpl exchangisJobDsBindService; @@ -304,161 +283,6 @@ public ExchangisJobVo updateJobContent(ExchangisJobVo jobVo) throws ExchangisJob return jobVo; } - @Override - public Message exportProject(Map params, String userName, HttpServletRequest request) throws ExchangisJobServerException, ServerException { - ExportedProject exportedProject = null; - Long projectId = Long.parseLong(params.get("projectId").toString()); - Boolean partial = (Boolean) params.get("partial"); - Map> moduleIdsMap = getModuleIdsMap(params); - - LOG.info("export project, user: {}, project: {}, partial:{}", userName, projectId, partial); - exportedProject = export(projectId, moduleIdsMap, partial, request); - String exported = null; - try { - exported = BDPJettyServerHelper.jacksonJson().writeValueAsString(exportedProject); - } catch (JsonProcessingException e) { - LOG.error("Occur error while tranform class", e.getMessage()); - } - - LOG.info("projectName: {}, exported:{}", exportedProject.getName(), exported); - BmlClient bmlClient = BmlClientFactory.createBmlClient(userName); - BmlUploadResponse bmlUploadResponse = bmlClient.uploadShareResource(userName, exportedProject.getName(), - "exchangis_exported_" + UUID.randomUUID(), new ByteArrayInputStream(exported.getBytes(StandardCharsets.UTF_8))); - - if (bmlUploadResponse == null || !bmlUploadResponse.isSuccess()) { - throw new ServerException("cannot upload exported data to BML"); - } - - LOG.info("{} is exporting the project, uploaded to BML the resourceID is {} and the version is {}", - userName, bmlUploadResponse.resourceId(), bmlUploadResponse.version()); - - Message message = Message.ok("export job") - .data("resourceId", bmlUploadResponse.resourceId()) - .data("version", bmlUploadResponse.version()); - return message; - } - - @Override - public ExportedProject export(Long projectId, Map> moduleIdsMap, boolean partial, HttpServletRequest request) throws ExchangisJobServerException { - ExportedProject exportedProject = new ExportedProject(); - ExchangisProjectInfo project = projectService.getProjectDetailById(projectId); - - LOG.info("execute export method! export project is {}.", project.getName()); - exportedProject.setName(project.getName()); - - setSqoop(projectId, moduleIdsMap, partial, exportedProject, request); - - setDatax(projectId, moduleIdsMap, partial, exportedProject, request); - - return exportedProject; - } - - private void setSqoop(Long projectId, Map> moduleIdsMap, boolean partial, ExportedProject exportedProject, HttpServletRequest request) throws ExchangisJobServerException { - List sqoops = new ArrayList<>(); - LOG.info("Request: {}", request); - if (partial) { - Set longs = moduleIdsMap.get(ModuleEnum.SQOOP_IDS.getName()); - if (longs.size() > 0) { - for (Long id : longs) { - LOG.info("id: {}", id); - ExchangisJobVo job = jobInfoService.getJob(id, false); - - String sqoopStr = null; - try { - sqoopStr = BDPJettyServerHelper.jacksonJson().writeValueAsString(job); - } catch (JsonProcessingException e) { - LOG.error("Occur error while tranform class", e.getMessage()); - } - - LOG.info("sqoopStr:{}", sqoopStr); - LOG.info("ExchangisJobVo sqoop: {}", job.getContent()); - LOG.info("getCreateTime: {}", job.getId()); - LOG.info("executorUser: {}", job.getExecuteUser()); - sqoops.add(job); - } - exportedProject.setSqoops(sqoops); - } - - } else { - LOG.info("Through request {} and projectId {} get Sqoopjob", request, projectId); - sqoops = jobInfoService.getSubJobList(request, projectId); - exportedProject.setSqoops(sqoops); - //exportedProject.setSqoops(jobInfoService.getByProject(request, projectId)); - } - LOG.info("exporting project, export sqoopJob: {}", exportedProject); - } - - private void setDatax(Long projectId, Map> moduleIdsMap, boolean partial, ExportedProject exportedProject, HttpServletRequest request) throws ExchangisJobServerException { - List dataxs = new ArrayList<>(); - LOG.info("Request: {}", request); - if (partial) { - Set longs = moduleIdsMap.get(ModuleEnum.DATAX_IDS.getName()); - if (longs.size() > 0) { - for (Long id : longs) { - LOG.info("id: {}", id); - ExchangisJobVo job = jobInfoService.getJob(id, false); - - String dataxStr = null; - try { - dataxStr = BDPJettyServerHelper.jacksonJson().writeValueAsString(job); - } catch (JsonProcessingException e) { - LOG.error("Occur error while tranform class", e.getMessage()); - } - - LOG.info("dataxStr:{}", dataxStr); - LOG.info("ExchangisJobVo sqoop: {}", job.getContent()); - LOG.info("getCreateTime: {}", job.getId()); - LOG.info("executorUser: {}", job.getExecuteUser()); - dataxs.add(job); - } - exportedProject.setDataxes(dataxs); - } - - } else { - LOG.info("Through request {} and projectId {} get Dataxjob", request, projectId); - dataxs = jobInfoService.getSubJobList(request, projectId); - exportedProject.setSqoops(dataxs); - } - LOG.info("exporting project, export dataxJob: {}", exportedProject); - - } - - /** - * 获取需要导出对象集合 - * - * @param params - * @return - */ - @Override - public Map> getModuleIdsMap(Map params) { - - Map> map = Maps.newHashMap(); - String sqoopIdsStr = null; - if(params.get("sqoopIds") != null) { - sqoopIdsStr = params.get("sqoopIds").toString(); - } - String dataxIdsStr = null; - if(params.get("dataXIds") != null) { - dataxIdsStr = params.get("dataXIds").toString(); - } - - Set sqoopIds = Sets.newHashSet(); - Set dataxIds = Sets.newHashSet(); - - if (StringUtils.isNotEmpty(sqoopIdsStr)) { - sqoopIds = Arrays.stream(StringUtils.split(sqoopIdsStr, ",")) - .map(Long::parseLong).collect(Collectors.toSet()); - } - if (StringUtils.isNotEmpty(dataxIdsStr)) { - dataxIds = Arrays.stream(StringUtils.split(dataxIdsStr, ",")) - .map(Long::parseLong).collect(Collectors.toSet()); - } - map.put("sqoopIds", sqoopIds); - map.put("dataXIds", dataxIds); - LOG.info("The objects to be exported are: {}", map); - return map; - } - @Override @Transactional(rollbackFor = Exception.class) public ExchangisJobVo copyJob(ExchangisJobVo jobVo) { diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java index b7c914e93..0a35fafcf 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java @@ -132,7 +132,6 @@ public void updateTaskStatus(LaunchedExchangisTask task, TaskStatus status, bool } @Override - @Transactional(rollbackFor = Exception.class) public void updateTaskProgress(LaunchedExchangisTask task, float progress) throws ExchangisOnEventException { task.setLastUpdateTime(Calendar.getInstance().getTime()); this.launchedTaskDao.upgradeLaunchedTaskProgress(task.getTaskId(), progress, task.getLastUpdateTime()); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java index d55be86f8..567895e08 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java @@ -26,14 +26,13 @@ public void updateJobDsBind(Long jobId, List dsBinds) { for (ExchangisJobDsBind dsBind : dsBinds) { this.dsBindMapper.insert(dsBind); } - } @Override public boolean inUse(Long datasourceId) { QueryWrapper condition = new QueryWrapper<>(); condition.eq("source_ds_id", datasourceId).or().eq("sink_ds_id", datasourceId); - Integer count = Optional.ofNullable(this.dsBindMapper.selectCount(condition)).orElse(0); + Long count = Optional.ofNullable(this.dsBindMapper.selectCount(condition)).orElse(0L); return count > 0; } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/JobAuthorityUtils.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/JobAuthorityUtils.java index a39d5dac0..29fd31713 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/JobAuthorityUtils.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/JobAuthorityUtils.java @@ -7,8 +7,8 @@ import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; import com.webank.wedatasphere.exchangis.job.server.mapper.ExchangisJobEntityDao; import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedJobDao; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectUser; -import com.webank.wedatasphere.exchangis.project.server.mapper.ProjectUserMapper; +import com.webank.wedatasphere.exchangis.project.entity.domain.ExchangisProjectUser; +import com.webank.wedatasphere.exchangis.project.provider.mapper.ProjectUserMapper; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; diff --git a/exchangis-job/exchangis-job-service/pom.xml b/exchangis-job/exchangis-job-service/pom.xml new file mode 100644 index 000000000..61a1b2133 --- /dev/null +++ b/exchangis-job/exchangis-job-service/pom.xml @@ -0,0 +1,33 @@ + + + + exchangis + com.webank.wedatasphere.exchangis + 1.1.2 + ../../pom.xml + + 4.0.0 + + exchangis-job-service + + + 8 + 8 + + + + + com.webank.wedatasphere.exchangis + exchangis-job-common + 1.1.1 + + + com.webank.wedatasphere.exchangis + exchangis-job-launcher + 1.1.1 + + + + \ No newline at end of file diff --git a/exchangis-job/pom.xml b/exchangis-job/pom.xml index 4a33a7154..e28c390ac 100644 --- a/exchangis-job/pom.xml +++ b/exchangis-job/pom.xml @@ -5,13 +5,13 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 exchangis-job pom - 1.1.1 + 1.1.2 exchangis-job-common diff --git a/exchangis-plugins/exchangis-appconn/pom.xml b/exchangis-plugins/exchangis-appconn/pom.xml index 98d291035..0c1212479 100644 --- a/exchangis-plugins/exchangis-appconn/pom.xml +++ b/exchangis-plugins/exchangis-appconn/pom.xml @@ -5,7 +5,7 @@ exchangis-plugins com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 ../pom.xml 4.0.0 diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/development/ExchangisExportOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/development/ExchangisExportOperation.java index 05e170013..f6dae2729 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/development/ExchangisExportOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/development/ExchangisExportOperation.java @@ -1,6 +1,5 @@ package com.webank.wedatasphere.exchangis.dss.appconn.operation.development; -import com.sun.xml.bind.v2.TODO; import com.webank.wedatasphere.dss.standard.app.development.operation.AbstractDevelopmentOperation; import com.webank.wedatasphere.dss.standard.app.development.operation.RefExportOperation; import com.webank.wedatasphere.dss.standard.app.development.ref.ExportResponseRef; diff --git a/exchangis-plugins/exchangis-appconn/src/main/resources/datax.icon b/exchangis-plugins/exchangis-appconn/src/main/resources/datax.icon new file mode 100644 index 000000000..bf047fb7b --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/resources/datax.icon @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql index a1eae0547..55105578c 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql +++ b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql @@ -6,7 +6,7 @@ delete from `dss_workspace_menu_appconn` where `appconn_id` in (select `id` fro delete from `dss_appconn` where `appconn_name`='exchangis'; INSERT INTO `dss_appconn` (`appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, `reference`, `class_name`, `appconn_class_path`, `resource`) -VALUES ('exchangis', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.exchangis.dss.appconn.ExchangisAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/exchangis/lib', ''); +VALUES ('exchangis', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.exchangis.dss.appconn.ExchangisAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/exchangis', ''); select @dss_appconn_exchangis_id:=id from `dss_appconn` where `appconn_name` = "exchangis"; INSERT INTO `dss_appconn_instance` (`appconn_id`, `label`, `url`, `enhance_json`, `homepage_uri`) @@ -18,20 +18,34 @@ INSERT INTO `dss_workspace_menu_appconn` (`appconn_id`, `menu_id`, `title_en`, ` VALUES(@dss_appconn_exchangis_id,@exchangis_menuId,'Exchangis','Exchangis','Exchangis','' ,'exchangis, statement','数据交换,数据源','1','enter Exchangis','进入Exchangis','user manual','用户手册','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/#/projectManage','shujukeshihua-logo',NULL,NULL,NULL,NULL,NULL,'shujukeshihua-icon'); -select @dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'; +-- Sqoop节点安装 +select @old_dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'; +select @old_dss_exchangis_dataxId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'; delete from `dss_workflow_node` where `node_type` like '%exchangis%'; +delete from `dss_workflow_node_to_group` where `node_id`=@old_dss_exchangis_sqoopId or `node_id`=@old_dss_exchangis_dataxId; +delete from `dss_workflow_node_to_ui` where `workflow_node_id`=@old_dss_exchangis_sqoopId or `workflow_node_id`=@old_dss_exchangis_dataxId; + +-- 节点表dss_workflow_node insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) values('sqoop','exchangis','linkis.appconn.exchangis.sqoop',1,'1','1','0','1','icons/sqoop.icon'); - --- 查找节点所属组的id select @dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'; -delete from `dss_workflow_node_to_group` where `node_id`=@dss_exchangis_sqoopId; -INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@dss_exchangis_sqoopId, 1); +insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon_path`) +values('datax','exchangis','linkis.appconn.exchangis.datax',1,'1','1','0','1','icons/datax.icon'); +select @dss_exchangis_dataxId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'; + +-- 节点组表dss_workflow_node_to_group +select @dss_workflow_node_group_id:=id from `dss_workflow_node_group` where `name` = '数据交换'; +INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@dss_exchangis_sqoopId, @dss_workflow_node_group_id); +INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@dss_exchangis_dataxId, @dss_workflow_node_group_id); --- 查找sqoop节点所属组的id -delete from `dss_workflow_node_to_ui` where `workflow_node_id`=@dss_exchangis_sqoopId; +-- 节点UI表dss_workflow_node_to_ui INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId, 1); INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId, 2); INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId, 3); INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId, 4); INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId, 5); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId, 1); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId, 2); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId, 3); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId, 4); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId, 5); diff --git a/exchangis-plugins/pom.xml b/exchangis-plugins/pom.xml index 486e5d4a0..8b290ca55 100644 --- a/exchangis-plugins/pom.xml +++ b/exchangis-plugins/pom.xml @@ -5,13 +5,13 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.1 + 1.1.2 4.0.0 exchangis-plugins pom - 1.1.1 + 1.1.2 exchangis-appconn diff --git a/exchangis-project/exchangis-project-entity/pom.xml b/exchangis-project/exchangis-project-entity/pom.xml new file mode 100644 index 000000000..198497921 --- /dev/null +++ b/exchangis-project/exchangis-project-entity/pom.xml @@ -0,0 +1,32 @@ + + + + exchangis-project + com.webank.wedatasphere.exchangis + 1.1.2 + + 4.0.0 + + exchangis-project-entity + + + 8 + 8 + + + + + com.webank.wedatasphere.exchangis + exchangis-dao + 1.1.2 + + + org.apache.commons + commons-math3 + 3.6.1 + + + + \ No newline at end of file diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProjectUser.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ExchangisProjectUser.java similarity index 95% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProjectUser.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ExchangisProjectUser.java index ff460da4f..195741fa6 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProjectUser.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ExchangisProjectUser.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.project.server.entity; +package com.webank.wedatasphere.exchangis.project.entity.domain; import java.util.Date; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/OperationType.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/OperationType.java similarity index 86% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/OperationType.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/OperationType.java index fcb958a6a..ab8f2ccf4 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/OperationType.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/OperationType.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.project.server.domain; +package com.webank.wedatasphere.exchangis.project.entity.domain; /** * @author jefftlin diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/ProjectPageQuery.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ProjectPageQuery.java similarity index 83% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/ProjectPageQuery.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ProjectPageQuery.java index 9a4a012f3..db42b536c 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/ProjectPageQuery.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/domain/ProjectPageQuery.java @@ -1,12 +1,12 @@ -package com.webank.wedatasphere.exchangis.project.server.domain; +package com.webank.wedatasphere.exchangis.project.entity.domain; import com.webank.wedatasphere.exchangis.common.pager.PageQuery; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; /** * For querying page */ public class ProjectPageQuery extends PageQuery { + /** * Project name */ diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProject.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/entity/ExchangisProject.java similarity index 98% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProject.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/entity/ExchangisProject.java index b0fdc0a6b..286c2ceb1 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProject.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/entity/ExchangisProject.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.project.server.entity; +package com.webank.wedatasphere.exchangis.project.entity.entity; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectAppVo.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectAppVo.java similarity index 97% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectAppVo.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectAppVo.java index f3ec21097..d6ca525c5 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectAppVo.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectAppVo.java @@ -1,8 +1,8 @@ -package com.webank.wedatasphere.exchangis.project.server.vo; +package com.webank.wedatasphere.exchangis.project.entity.vo; import com.fasterxml.jackson.annotation.JsonAlias; import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.entity.entity.ExchangisProject; import org.hibernate.validator.constraints.NotBlank; import javax.validation.constraints.NotNull; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectInfo.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectInfo.java similarity index 96% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectInfo.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectInfo.java index fae790d63..dc356715b 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectInfo.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectInfo.java @@ -1,9 +1,8 @@ -package com.webank.wedatasphere.exchangis.project.server.vo; +package com.webank.wedatasphere.exchangis.project.entity.vo; import com.fasterxml.jackson.annotation.JsonAlias; -import com.fasterxml.jackson.annotation.JsonInclude; import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.entity.entity.ExchangisProject; import org.hibernate.validator.constraints.NotBlank; import javax.validation.constraints.NotNull; @@ -18,7 +17,7 @@ * TODO @JsonInclude(JsonInclude.Include.NON_EMPTY) */ //@JsonInclude(JsonInclude.Include.NON_EMPTY) -public class ExchangisProjectInfo { +public class ExchangisProjectInfo { /** * ID */ diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectUserVo.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectUserVo.java similarity index 90% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectUserVo.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectUserVo.java index b959a5dc3..fef892115 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectUserVo.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ExchangisProjectUserVo.java @@ -1,9 +1,8 @@ -package com.webank.wedatasphere.exchangis.project.server.vo; +package com.webank.wedatasphere.exchangis.project.entity.vo; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectUser; +import com.webank.wedatasphere.exchangis.project.entity.domain.ExchangisProjectUser; import javax.validation.constraints.NotNull; -import java.util.Date; /** * @author tikazhang diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ProjectQueryVo.java similarity index 73% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java rename to exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ProjectQueryVo.java index 7f56cc0f5..42498c95f 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java +++ b/exchangis-project/exchangis-project-entity/src/main/java/com/webank/wedatasphere/exchangis/project/entity/vo/ProjectQueryVo.java @@ -1,7 +1,7 @@ -package com.webank.wedatasphere.exchangis.project.server.vo; +package com.webank.wedatasphere.exchangis.project.entity.vo; import com.fasterxml.jackson.annotation.JsonInclude; -import com.webank.wedatasphere.exchangis.project.server.domain.ProjectPageQuery; +import com.webank.wedatasphere.exchangis.project.entity.domain.ProjectPageQuery; import java.util.Map; import java.util.Optional; @@ -15,6 +15,7 @@ public class ProjectQueryVo extends ProjectPageQuery { public ProjectQueryVo(){ } + private Map labels; public Map getLabels() { @@ -31,4 +32,9 @@ public ProjectQueryVo(String name, Integer current, Integer size){ this.size = Optional.ofNullable(size).orElse(10); } + public ProjectQueryVo(Map labels, String name){ + this.labels = labels; + this.name = name; + } + } diff --git a/exchangis-project/exchangis-project-provider/pom.xml b/exchangis-project/exchangis-project-provider/pom.xml new file mode 100644 index 000000000..b658e8b07 --- /dev/null +++ b/exchangis-project/exchangis-project-provider/pom.xml @@ -0,0 +1,38 @@ + + + + exchangis-project + com.webank.wedatasphere.exchangis + 1.1.2 + + 4.0.0 + + exchangis-project-provider + + + 8 + 8 + + + + + com.webank.wedatasphere.exchangis + exchangis-project-entity + 1.1.2 + + + + + + + src/main/java + + **/*.xml + + + + + + \ No newline at end of file diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectMapper.java b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectMapper.java similarity index 90% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectMapper.java rename to exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectMapper.java index cbdc53bdb..4c7b7f41a 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectMapper.java +++ b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectMapper.java @@ -1,7 +1,7 @@ -package com.webank.wedatasphere.exchangis.project.server.mapper; +package com.webank.wedatasphere.exchangis.project.provider.mapper; -import com.webank.wedatasphere.exchangis.project.server.domain.ProjectPageQuery; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.entity.domain.ProjectPageQuery; +import com.webank.wedatasphere.exchangis.project.entity.entity.ExchangisProject; import org.apache.ibatis.annotations.Param; import java.util.List; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectUserMapper.java b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectUserMapper.java similarity index 85% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectUserMapper.java rename to exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectUserMapper.java index 8a5a09c85..33cac2930 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectUserMapper.java +++ b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/ProjectUserMapper.java @@ -1,6 +1,6 @@ -package com.webank.wedatasphere.exchangis.project.server.mapper; +package com.webank.wedatasphere.exchangis.project.provider.mapper; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectUser; +import com.webank.wedatasphere.exchangis.project.entity.domain.ExchangisProjectUser; import org.apache.ibatis.annotations.Param; import java.util.List; diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectMapper.xml similarity index 96% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml rename to exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectMapper.xml index df73890be..dbc90c879 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml +++ b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectMapper.xml @@ -1,9 +1,9 @@ - + - + @@ -33,7 +33,7 @@ - AND dp.name like concat('%', #{name}, '%') escape '/' + AND dp.name like concat('%', #{name}, '%') AND dp.domain = #{domain} @@ -118,7 +118,7 @@ diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectUserMapper.xml b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectUserMapper.xml similarity index 90% rename from exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectUserMapper.xml rename to exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectUserMapper.xml index 6335cd8fe..f4b3709a6 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectUserMapper.xml +++ b/exchangis-project/exchangis-project-provider/src/main/java/com/webank/wedatasphere/exchangis/project/provider/mapper/impl/ProjectUserMapper.xml @@ -1,9 +1,9 @@ - + - + @@ -22,7 +22,7 @@ delete from - where project_id = #{projectId}; + where project_id = #{projectId}