From 081feb6b0baf6fb67b2c109f7ee86a3f947168b6 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Mon, 11 Jul 2022 09:56:07 +0800 Subject: [PATCH 001/234] fix exchangis_ddl.sql --- db/exchangis_ddl.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/exchangis_ddl.sql b/db/exchangis_ddl.sql index 0ec5466f8..e765f50ee 100644 --- a/db/exchangis_ddl.sql +++ b/db/exchangis_ddl.sql @@ -90,7 +90,7 @@ CREATE TABLE `exchangis_project_user` ( `priv` int(20) DEFAULT NULL, `last_update_time` datetime DEFAULT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT +) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT; -- exchangis_v4.exchangis_launchable_task definition DROP TABLE IF EXISTS `exchangis_launchable_task`; From 9a91eebac8de153120b2bb0f6d0c1ada3c51df7b Mon Sep 17 00:00:00 2001 From: FinalTarget <1326477975@qq.com> Date: Tue, 19 Jul 2022 17:37:53 +0800 Subject: [PATCH 002/234] Fix updateProjectOperation fail bug. --- db/exchangis_ddl.sql | 2 +- .../execute/ExchangisJobExecuteRestfulApi.java | 2 +- .../project/ExchangisProjectUpdateOperation.java | 12 ++++++++++++ .../ExchangisProjectDssAppConnRestfulApi.java | 4 ++-- .../server/service/impl/ProjectServiceImpl.java | 4 ++-- .../src/main/resources/exchangis.properties | 4 ++-- 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/db/exchangis_ddl.sql b/db/exchangis_ddl.sql index 0ec5466f8..e765f50ee 100644 --- a/db/exchangis_ddl.sql +++ b/db/exchangis_ddl.sql @@ -90,7 +90,7 @@ CREATE TABLE `exchangis_project_user` ( `priv` int(20) DEFAULT NULL, `last_update_time` datetime DEFAULT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT +) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT; -- exchangis_v4.exchangis_launchable_task definition DROP TABLE IF EXISTS `exchangis_launchable_task`; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java index 201190671..e90b60c4b 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java @@ -96,7 +96,7 @@ public Message getExecutedJobTaskList(@PathVariable(value = "jobExecutionId") St LOG.error(errorMessage, e); message = Message.error(message + "(执行任务出错), reason: " + e.getMessage()); } - message.setMethod("/api/rest_j/v1/dss/exchangis/main/job/execution/" + jobExecutionId + "/taskList"); + message.setMethod("/api/rest_j/v1/" + jobExecutionId + "/taskList"); return message; } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectUpdateOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectUpdateOperation.java index a54c6f58c..a90cebdea 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectUpdateOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectUpdateOperation.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.dss.appconn.operation.project; +import com.webank.wedatasphere.dss.common.utils.MapUtils; import com.webank.wedatasphere.dss.standard.app.sso.origin.request.action.DSSPutAction; import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureOperation; import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectUpdateOperation; @@ -29,6 +30,8 @@ public ResponseRef updateProject(ProjectUpdateRequestRef.ProjectUpdateRequestRef projectRequestRef.getRefProjectId(), url); DSSPutAction putAction = new DSSPutAction(); putAction.setUser(projectRequestRef.getUserName()); + addProjectInfo(putAction, projectRequestRef); + logger.info("project payload is: {}", putAction.getRequestPayload()); ExchangisProjectCreationOperation.addProjectInfo(putAction, projectRequestRef); InternalResponseRef responseRef = ExchangisHttpUtils.getResponseRef(projectRequestRef, url, putAction, ssoRequestOperation); logger.info("User {} updated Exchangis project {} with response {}.", projectRequestRef.getUserName(), projectRequestRef.getRefProjectId(), responseRef.getResponseBody()); @@ -45,4 +48,13 @@ public void init() { super.init(); projectUpdateUrl = mergeBaseUrl(mergeUrl(API_REQUEST_PREFIX, "appProject")); } + + public static void addProjectInfo(DSSPutAction putAction, ProjectUpdateRequestRef requestRef) { + putAction.addRequestPayload("id", requestRef.getRefProjectId()); + putAction.addRequestPayload("projectName", requestRef.getDSSProject().getName()); + putAction.addRequestPayload("description", requestRef.getDSSProject().getDescription()); + putAction.addRequestPayload("domain", Constraints.DOMAIN_NAME); + putAction.addRequestPayload("source", MapUtils.newCommonMap("workspace", requestRef.getWorkspace().getWorkspaceName())); + } + } diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java index 46253968d..03664bc31 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java @@ -87,10 +87,10 @@ public Message updateProject(@PathVariable("id") Long id, @Validated({UpdateGrou // return Message.error("You have no permission to update (没有编辑权限，无法更新项目)"); // } String domain = projectStored.getDomain(); - if (StringUtils.isNotBlank(domain) && !ExchangisProject.Domain.STANDALONE.name() + /*if (StringUtils.isNotBlank(domain) && !ExchangisProject.Domain.STANDALONE.name() .equalsIgnoreCase(domain)){ return Message.error("Cannot update the outer project (无法更新来自 " + domain + " 的外部项目)"); - } + }*/ LOG.info("UpdateProject vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); projectService.updateProject(projectVo, username); return ExchangisProjectRestfulUtils.dealOk("更新工程成功", diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectServiceImpl.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectServiceImpl.java index e28df5556..1500eeca3 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectServiceImpl.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectServiceImpl.java @@ -125,7 +125,7 @@ public void updateProject(ExchangisProjectInfo projectVo, String userName) { updatedProject.setLastUpdateTime(Calendar.getInstance().getTime()); this.projectMapper.updateOne(updatedProject); - List projectUsers = new ArrayList<>(); + /*List projectUsers = new ArrayList<>(); if (updatedProject.getViewUsers() != null && updatedProject.getViewUsers().length()!=0) { for (String view : updatedProject.getViewUsers().split(",")) { ExchangisProjectUser projectUser = new ExchangisProjectUser(); @@ -162,7 +162,7 @@ public void updateProject(ExchangisProjectInfo projectVo, String userName) { projectUser.setPriv(0); projectUser.setUpdateTime(updatedProject.getLastUpdateTime()); projectUsers.add(projectUser); - this.projectUserMapper.updateProjectUser(projectUsers); + this.projectUserMapper.updateProjectUser(projectUsers);*/ } @Override diff --git a/exchangis-server/src/main/resources/exchangis.properties b/exchangis-server/src/main/resources/exchangis.properties index d04af6fd8..0b906a666 100644 --- a/exchangis-server/src/main/resources/exchangis.properties +++ b/exchangis-server/src/main/resources/exchangis.properties @@ -20,9 +20,9 @@ wds.linkis.test.mode=true wds.linkis.server.mybatis.datasource.url=jdbc:mysql://localhost:3306/exchangis_v3?useSSL=false&characterEncoding=UTF-8 -wds.linkis.server.mybatis.datasource.username=root +wds.linkis.server.mybatis.datasource.username= -wds.linkis.server.mybatis.datasource.password=123456 +wds.linkis.server.mybatis.datasource.password= wds.linkis.log.clear=true From 5051cc4fb4407e36504b4b007e262b1c405dd297 Mon Sep 17 00:00:00 2001 From: FinalTarget <1326477975@qq.com> Date: Wed, 20 Jul 2022 18:00:10 +0800 Subject: [PATCH 003/234] Fix ExchangisJobExecuteRestfulApi and ExchangisJobRestfulApi authrity bug. --- .../restful/ExchangisJobRestfulApi.java | 34 ++++++++++++++- .../ExchangisJobExecuteRestfulApi.java | 41 ++++++++++++++++--- .../exchangis/datax/common/CryptoUtils.java | 2 +- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java index 332dab23f..5fbd3aeaa 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java @@ -9,6 +9,8 @@ import com.webank.wedatasphere.exchangis.job.enums.EngineTypeEnum; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; +import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; @@ -38,6 +40,9 @@ public class ExchangisJobRestfulApi { @Resource private JobInfoService jobInfoService; + @Resource + private ProjectService projectService; + /** * Query job in page * @@ -60,6 +65,11 @@ public Message getJobList(@RequestParam(value = "projectId") Long projectId, projectId, jobType, name, current, size ); String userName = SecurityFilter.getLoginUsername(request); + + ExchangisProjectInfo projectStored = projectService.getProjectById(projectId); + if (!hasProjectAuthority(userName, projectStored)) { + return Message.error("You have no permission to create Job (没有创建任务权限)"); + } queryVo.setCreateUser(userName); try { PageResult pageResult = jobInfoService.queryJobList(queryVo); @@ -100,6 +110,12 @@ public Message createJob( String userName = SecurityFilter.getLoginUsername(request); exchangisJobVo.setCreateUser(userName); Message response = Message.ok(); + + ExchangisProjectInfo projectStored = projectService.getProjectById(exchangisJobVo.getProjectId()); + if (!hasProjectAuthority(userName, projectStored)) { + return Message.error("You have no permission to create Job (没有创建任务权限)"); + } + /* if (!AuthorityUtils.hasOwnAuthority(exchangisJobVo.getProjectId(), userName) && !AuthorityUtils.hasEditAuthority(exchangisJobVo.getProjectId(), userName)) { return Message.error("You have no permission to create (没有编辑权限，无法创建任务)"); }*/ @@ -192,7 +208,7 @@ public Message deleteJob(@PathVariable("id") Long id, HttpServletRequest request Message response = Message.ok("job deleted"); try { if (!hasAuthority(userName, jobInfoService.getJob(id, true))) { - return Message.error("You have no permission to update ()"); + return Message.error("You have no permission to delete (没有删除权限)"); } /* if (!AuthorityUtils.hasOwnAuthority(jobInfoService.getJob(id, true).getProjectId(), userName) && !AuthorityUtils.hasEditAuthority(jobInfoService.getJob(id, true).getProjectId(), userName)) { return Message.error("You have no permission to delete (没有编辑权限，无法删除)"); @@ -219,6 +235,11 @@ public Message getJob(HttpServletRequest request, @PathVariable("id") Long id) { Message response = Message.ok(); try { LOG.info("Request88888: {}", request); + + String userName = SecurityFilter.getLoginUsername(request); + if (!hasAuthority(userName, jobInfoService.getJob(id, true))) { + return Message.error("You have no permission to get job (没有获取任务权限)"); + } ExchangisJobVo job = jobInfoService.getDecoratedJob(request, id); response.data("result", job); } catch (Exception e) { @@ -292,10 +313,14 @@ public Message saveJobConfig(@PathVariable("id") Long id, jobVo.setId(id); jobVo.setModifyUser(SecurityFilter.getLoginUsername(request)); Message response = Message.ok(); + String loginUser = SecurityFilter.getLoginUsername(request); try { /*if (!AuthorityUtils.hasOwnAuthority(jobVo.getProjectId(), jobVo.getModifyUser()) && !AuthorityUtils.hasEditAuthority(jobVo.getProjectId(), jobVo.getModifyUser())) { return Message.error("You have no permission to update (没有编辑权限)"); }*/ + if (!hasAuthority(loginUser, jobInfoService.getJob(id, true))){ + return Message.error("You have no permission to save content (没有保存任务权限)"); + } ExchangisJobVo exchangisJob = jobInfoService.updateJobConfig(jobVo); response.data("id", exchangisJob.getId()); } catch (Exception e) { @@ -318,6 +343,9 @@ public Message saveSubJobs(@PathVariable("id") Long id, /*if (!AuthorityUtils.hasOwnAuthority(projectId, loginUser) && !AuthorityUtils.hasEditAuthority(projectId, loginUser)) { return Message.error("You have no permission to update (没有编辑权限，无法保存配置)"); }*/ + if (!hasAuthority(loginUser, jobInfoService.getJob(id, true))){ + return Message.error("You have no permission to save content (没有保存任务权限)"); + } ExchangisJobVo exchangisJob = jobInfoService.updateJobContent(jobVo); response.data("id", exchangisJob.getId()); } catch (Exception e) { @@ -343,4 +371,8 @@ private boolean hasAuthority(String username, ExchangisJobVo job) { return Objects.nonNull(job) && username.equals(job.getCreateUser()); } + private boolean hasProjectAuthority(String username, ExchangisProjectInfo project){ + return Objects.nonNull(project) && username.equals(project.getCreateUser()); + } + } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java index e90b60c4b..77778e91b 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java @@ -86,9 +86,13 @@ public Message executeJob(@RequestBody(required = false) Map per } @RequestMapping( value = "/execution/{jobExecutionId}/taskList", method = RequestMethod.GET) - public Message getExecutedJobTaskList(@PathVariable(value = "jobExecutionId") String jobExecutionId) { + public Message getExecutedJobTaskList(@PathVariable(value = "jobExecutionId") String jobExecutionId, HttpServletRequest request) { Message message = Message.ok("Submitted succeed(提交成功)！"); + String userName = SecurityFilter.getLoginUsername(request); try { + if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + return Message.error("You have no permission to get taskList (没有获取任务列表权限)"); + } List jobTaskList = executeService.getExecutedJobTaskList(jobExecutionId); message.data("tasks", jobTaskList); } catch (ExchangisJobServerException e) { @@ -101,9 +105,14 @@ public Message getExecutedJobTaskList(@PathVariable(value = "jobExecutionId") St } @RequestMapping( value = "/execution/{jobExecutionId}/progress", method = RequestMethod.GET) - public Message getExecutedJobAndTaskStatus(@PathVariable(value = "jobExecutionId") String jobExecutionId) { + public Message getExecutedJobAndTaskStatus(@PathVariable(value = "jobExecutionId") String jobExecutionId, HttpServletRequest request) { ExchangisJobProgressVo jobAndTaskStatus; + String userName = SecurityFilter.getLoginUsername(request); try { + + if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + return Message.error("You have no permission to get task progress (没有获取任务进度权限)"); + } jobAndTaskStatus = executeService.getExecutedJobProgressInfo(jobExecutionId); } catch (ExchangisJobServerException e) { // TODO Log exception @@ -116,9 +125,13 @@ public Message getExecutedJobAndTaskStatus(@PathVariable(value = "jobExecutionId } @RequestMapping( value = "/execution/{jobExecutionId}/status", method = RequestMethod.GET) - public Message getExecutedJobStatus(@PathVariable(value = "jobExecutionId") String jobExecutionId) { + public Message getExecutedJobStatus(@PathVariable(value = "jobExecutionId") String jobExecutionId, HttpServletRequest request) { Message message = Message.ok("Submitted succeed(提交成功)！"); + String userName = SecurityFilter.getLoginUsername(request); try { + if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + return Message.error("You have no permission to get tastStatus (没有权限去获取任务状态)"); + } ExchangisJobProgressVo jobStatus = executeService.getJobStatus(jobExecutionId); message.setMethod("/api/rest_j/v1/dss/exchangis/main/job/execution/" + jobExecutionId + "/status"); message.data("status", jobStatus.getStatus()); @@ -143,7 +156,11 @@ public Message getJobExecutionLogs(@PathVariable(value = "jobExecutionId") Strin Message result = Message.ok("Submitted succeed(提交成功)！"); LogQuery logQuery = new LogQuery(fromLine, pageSize, ignoreKeywords, onlyKeywords, lastRows); + String userName = SecurityFilter.getLoginUsername(request); try { + if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + return Message.error("You have no permission to get logs (没有获取任务日志权限)"); + } ExchangisCategoryLogVo categoryLogVo = this.executeService .getJobLogInfo(jobExecutionId, logQuery, SecurityFilter.getLoginUsername(request)); result.setData(Json.convert(categoryLogVo, Map.class, String.class, Object.class)); @@ -157,10 +174,14 @@ public Message getJobExecutionLogs(@PathVariable(value = "jobExecutionId") Strin } @RequestMapping( value = "/execution/{jobExecutionId}/kill", method = RequestMethod.POST) - public Message ExecutedJobKill(@PathVariable(value = "jobExecutionId") String jobExecutionId) throws ExchangisJobServerException { + public Message ExecutedJobKill(@PathVariable(value = "jobExecutionId") String jobExecutionId, HttpServletRequest request) throws ExchangisJobServerException { ExchangisJobProgressVo jobStatus = executeService.getJobStatus(jobExecutionId); String status = jobStatus.getStatus().toString(); Message message = null; + String userName = SecurityFilter.getLoginUsername(request); + if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + return Message.error("You have no permission to get tastStatus (没有权限去获取任务状态)"); + } if (!TaskStatus.isCompleted(jobStatus.getStatus())) { message = Message.ok("Kill succeed(停止成功)！"); @@ -215,10 +236,14 @@ private boolean hasAuthority(String username, ExchangisJobInfo jobInfo){ } @RequestMapping( value = "/{jobExecutionId}/deleteJob", method = RequestMethod.POST) - public Message ExecutedJobDelete(@PathVariable(value = "jobExecutionId") String jobExecutionId) throws ExchangisJobServerException { + public Message ExecutedJobDelete(@PathVariable(value = "jobExecutionId") String jobExecutionId, HttpServletRequest request) throws ExchangisJobServerException { //ExchangisLaunchedJobEntity jobAndTaskStatus = exchangisExecutionService.getExecutedJobAndTaskStatus(jobExecutionId); Message message = Message.ok("Kill succeed(停止成功)！"); + String userName = SecurityFilter.getLoginUsername(request); try { + if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + return Message.error("You have no permission to delete this record (没有删除历史记录权限)"); + } executeService.deleteJob(jobExecutionId); message.data("jobExecutionId", jobExecutionId); } catch (ExchangisJobServerException e){ @@ -231,10 +256,14 @@ public Message ExecutedJobDelete(@PathVariable(value = "jobExecutionId") String } @RequestMapping( value = "/{jobExecutionId}/allTaskStatus", method = RequestMethod.GET) - public Message allTaskStatus(@PathVariable(value = "jobExecutionId") String jobExecutionId) throws ExchangisJobServerException { + public Message allTaskStatus(@PathVariable(value = "jobExecutionId") String jobExecutionId, HttpServletRequest request) throws ExchangisJobServerException { //ExchangisLaunchedJobEntity jobAndTaskStatus = exchangisExecutionService.getExecutedJobAndTaskStatus(jobExecutionId); Message message = Message.ok("所有任务状态"); + String userName = SecurityFilter.getLoginUsername(request); try { + if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + return Message.error("You have no permission to get tastStatus (没有权限去获取任务状态)"); + } List allStatus = executeService.allTaskStatus(jobExecutionId); message.data("allStatus", allStatus); message.data("jobExecutionId", jobExecutionId); diff --git a/exchangis-plugins/engine/datax/datax-core/src/main/java/com/webank/wedatasphere/exchangis/datax/common/CryptoUtils.java b/exchangis-plugins/engine/datax/datax-core/src/main/java/com/webank/wedatasphere/exchangis/datax/common/CryptoUtils.java index 2e1c6a180..c9037c15b 100644 --- a/exchangis-plugins/engine/datax/datax-core/src/main/java/com/webank/wedatasphere/exchangis/datax/common/CryptoUtils.java +++ b/exchangis-plugins/engine/datax/datax-core/src/main/java/com/webank/wedatasphere/exchangis/datax/common/CryptoUtils.java @@ -74,7 +74,7 @@ public static Object string2Object(String str) throws IOException, ClassNotFound public static String md5(String source, String salt, int iterator){ StringBuilder token = new StringBuilder(); try{ - MessageDigest digest = MessageDigest.getInstance("md5"); + MessageDigest digest = MessageDigest.getInstance("sha-256"); if(StringUtils.isNotEmpty(salt)){ digest.update(salt.getBytes(StandardCharsets.UTF_8)); } From f4ca23fbe333ff269261af8bdc99f1487ab92bc0 Mon Sep 17 00:00:00 2001 From: FinalTarget <1326477975@qq.com> Date: Wed, 20 Jul 2022 20:51:42 +0800 Subject: [PATCH 004/234] Remove sensitive information from logs --- .../log/service/LocalSimpleJobLogService.java | 7 ++++++- .../execute/ExchangisJobExecuteRestfulApi.java | 14 +++++++------- .../service/impl/DefaultJobExecuteService.java | 7 +++++++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java index 5b4a991dc..a0f843e86 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java @@ -170,7 +170,12 @@ public LogResult logsFromPageAndPath(String logPath, LogQuery logQuery) { rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); } if (!rowIgnore) { - logs.add(new String(line.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8)); + if (line.contains("password")) { + LOG.info("have error information66666"); + } + if (!line.contains("password")) { + logs.add(new String(line.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8)); + } readLine += 1; } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java index 77778e91b..7e11e0900 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java @@ -90,7 +90,7 @@ public Message getExecutedJobTaskList(@PathVariable(value = "jobExecutionId") St Message message = Message.ok("Submitted succeed(提交成功)！"); String userName = SecurityFilter.getLoginUsername(request); try { - if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + if(!executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { return Message.error("You have no permission to get taskList (没有获取任务列表权限)"); } List jobTaskList = executeService.getExecutedJobTaskList(jobExecutionId); @@ -110,7 +110,7 @@ public Message getExecutedJobAndTaskStatus(@PathVariable(value = "jobExecutionId String userName = SecurityFilter.getLoginUsername(request); try { - if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + if(!executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { return Message.error("You have no permission to get task progress (没有获取任务进度权限)"); } jobAndTaskStatus = executeService.getExecutedJobProgressInfo(jobExecutionId); @@ -129,7 +129,7 @@ public Message getExecutedJobStatus(@PathVariable(value = "jobExecutionId") Stri Message message = Message.ok("Submitted succeed(提交成功)！"); String userName = SecurityFilter.getLoginUsername(request); try { - if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + if(!executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { return Message.error("You have no permission to get tastStatus (没有权限去获取任务状态)"); } ExchangisJobProgressVo jobStatus = executeService.getJobStatus(jobExecutionId); @@ -158,7 +158,7 @@ public Message getJobExecutionLogs(@PathVariable(value = "jobExecutionId") Strin ignoreKeywords, onlyKeywords, lastRows); String userName = SecurityFilter.getLoginUsername(request); try { - if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + if(!executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { return Message.error("You have no permission to get logs (没有获取任务日志权限)"); } ExchangisCategoryLogVo categoryLogVo = this.executeService @@ -179,7 +179,7 @@ public Message ExecutedJobKill(@PathVariable(value = "jobExecutionId") String jo String status = jobStatus.getStatus().toString(); Message message = null; String userName = SecurityFilter.getLoginUsername(request); - if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + if(!executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { return Message.error("You have no permission to get tastStatus (没有权限去获取任务状态)"); } if (!TaskStatus.isCompleted(jobStatus.getStatus())) @@ -241,7 +241,7 @@ public Message ExecutedJobDelete(@PathVariable(value = "jobExecutionId") String Message message = Message.ok("Kill succeed(停止成功)！"); String userName = SecurityFilter.getLoginUsername(request); try { - if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + if(!executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { return Message.error("You have no permission to delete this record (没有删除历史记录权限)"); } executeService.deleteJob(jobExecutionId); @@ -261,7 +261,7 @@ public Message allTaskStatus(@PathVariable(value = "jobExecutionId") String jobE Message message = Message.ok("所有任务状态"); String userName = SecurityFilter.getLoginUsername(request); try { - if(executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { + if(!executeService.hasExecuteJobAuthority(jobExecutionId, userName)) { return Message.error("You have no permission to get tastStatus (没有权限去获取任务状态)"); } List allStatus = executeService.allTaskStatus(jobExecutionId); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java index 3fea2972e..6592d9c23 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java @@ -332,6 +332,13 @@ public String executeJob(ExchangisJobInfo jobInfo, String execUser) throws Excha private ExchangisCategoryLogVo resultToCategoryLog(LogQuery logQuery, LogResult logResult, TaskStatus status){ ExchangisCategoryLogVo categoryLogVo = new ExchangisCategoryLogVo(); boolean noLogs = logResult.getLogs().isEmpty(); + for (int i = 0; i < logResult.getLogs().size(); i++) { + if (logResult.getLogs().get(i).contains("password")) { + LOG.info("Sensitive information in there: {}", logResult.getLogs().get(i)); + logResult.getLogs().set(i, "----"); + LOG.info("Change line is: {}", logResult.getLogs().get(i)); + } + } if (Objects.nonNull(logQuery.getLastRows())){ logResult.setEnd(true); }else if (noLogs){ From ba59b6ddd2b1d35d5a64ac43bd55cc1cc908d265 Mon Sep 17 00:00:00 2001 From: FinalTarget <1326477975@qq.com> Date: Thu, 21 Jul 2022 15:20:29 +0800 Subject: [PATCH 005/234] Mybatis sql inject fix. log permission added --- .../job/server/mapper/impl/ExchangisLaunchTaskMapper.xml | 2 +- .../job/server/mapper/impl/LaunchableTaskMapper.xml | 2 +- .../job/server/mapper/impl/LaunchedJobMapper.xml | 4 ++-- .../restful/execute/ExchangisJobExecuteRestfulApi.java | 2 +- .../restful/execute/ExchangisTaskExecuteRestfulApi.java | 9 +++++++++ .../server/service/impl/DefaultJobExecuteService.java | 8 +++++--- 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisLaunchTaskMapper.xml b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisLaunchTaskMapper.xml index 05e00f30c..c65797ef7 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisLaunchTaskMapper.xml +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisLaunchTaskMapper.xml @@ -52,7 +52,7 @@ order by create_time desc - limit ${start}, ${size} + limit #{start}, #{size} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchableTaskMapper.xml b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchableTaskMapper.xml index 684bdab1c..7770e9b6d 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchableTaskMapper.xml +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchableTaskMapper.xml @@ -80,7 +80,7 @@