From 588fa0d1af343fe76f66e21a09d9dee2f9fde776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Mon, 16 Dec 2024 16:30:18 +0800 Subject: [PATCH 01/15] Code Optimization --- .../linkis/ujes/jdbc/UJESSQLResultSet.scala | 30 +++++++++++-------- .../io/executor/IoEngineConnExecutor.scala | 5 +++- .../engineplugin/io/utils/IOHelp.scala | 18 +++++------ 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala index 37c379a7ca..7a3fdf0f7e 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala @@ -175,13 +175,23 @@ class UJESSQLResultSet( if (null == resultSetResult) { return } - metaData = resultSetResult.getMetadata.asInstanceOf[util.List[util.Map[String, String]]] - if (null != metaData) { - for (cursor <- 1 to metaData.size()) { - val col = metaData.get(cursor - 1) - resultSetMetaData.setColumnNameProperties(cursor, col.get("columnName")) - resultSetMetaData.setDataTypeProperties(cursor, col.get("dataType")) - resultSetMetaData.setCommentPropreties(cursor, col.get("comment")) + val metaTmp = resultSetResult.getMetadata + if (NULL_VALUE.equals(String.valueOf(metaTmp))) { + val fileContentList = resultSetResult.getFileContent.asInstanceOf[util.List[util.List[String]]] + if (null != fileContentList) { + resultSetMetaData.setColumnNameProperties(1, "linkis_string") + resultSetMetaData.setDataTypeProperties(1, "String") + resultSetMetaData.setCommentPropreties(1, NULL_VALUE) + } + } else { + metaData = metaTmp.asInstanceOf[util.List[util.Map[String, String]]] + if (null != metaData) { + for (cursor <- 1 to metaData.size()) { + val col = metaData.get(cursor - 1) + resultSetMetaData.setColumnNameProperties(cursor, col.get("columnName")) + resultSetMetaData.setDataTypeProperties(cursor, col.get("dataType")) + resultSetMetaData.setCommentPropreties(cursor, col.get("comment")) + } } } } @@ -194,12 +204,6 @@ class UJESSQLResultSet( resultSetResult.getFileContent.asInstanceOf[util.ArrayList[util.ArrayList[String]]] } - def getResultSet(): util.ArrayList[util.ArrayList[String]] = { - resultSetResultInit() - resultSetInit() - resultSetRow - } - private def init(): Unit = { resultSetResultInit() metaDataInit() diff --git a/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/executor/IoEngineConnExecutor.scala b/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/executor/IoEngineConnExecutor.scala index e8feb0d354..a9ec71ba00 100644 --- a/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/executor/IoEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/executor/IoEngineConnExecutor.scala @@ -60,6 +60,8 @@ import java.util.concurrent.atomic.AtomicLong import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer +import com.google.gson.internal.LinkedTreeMap + class IoEngineConnExecutor(val id: Int, val outputLimit: Int = 10) extends ConcurrentComputationExecutor(outputLimit) with Logging { @@ -322,7 +324,8 @@ class IoEngineConnExecutor(val id: Int, val outputLimit: Int = 10) s"Creator ${methodEntity.creatorUser} for user ${methodEntity.proxyUser} init fs $methodEntity" ) var fsId = methodEntity.id - val properties = methodEntity.params(0).asInstanceOf[Map[String, String]] + val properties = + methodEntity.params(0).asInstanceOf[LinkedTreeMap[String, String]].asScala.toMap val proxyUser = methodEntity.proxyUser if (!fsProxyService.canProxyUser(methodEntity.creatorUser, proxyUser, methodEntity.fsType)) { throw new StorageErrorException( diff --git a/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/utils/IOHelp.scala b/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/utils/IOHelp.scala index b6f35a2e8e..8d84a1b0b0 100644 --- a/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/utils/IOHelp.scala +++ b/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/utils/IOHelp.scala @@ -61,21 +61,17 @@ object IOHelp { writer.addRecord(ioRecord) writer.toString() } else if (method.params.length == 3) { - val position = - if (method.params(1).toString.toInt < 0) { - 0 - } else { - method.params(1).toString.toInt - } + val parm1 = BigDecimal(method.params(1).toString).intValue() + val parm2 = BigDecimal(method.params(2).toString).intValue() + if (parm1 > 0) { + inputStream.skip(parm1) + } val fetchSize = - if (method.params(2).toString.toInt > maxPageSize) { + if (parm2 > maxPageSize) { maxPageSize.toInt } else { - method.params(2).toString.toInt + parm2 } - if (position > 0) { - inputStream.skip(position) - } val bytes = new Array[Byte](fetchSize) val len = StorageUtils.readBytes(inputStream, bytes, fetchSize) val ioMetaData = new IOMetaData(0, len) From 804adfa7a18ec8aa3dbe2d90bf6d19d133cea16f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Mon, 16 Dec 2024 17:38:19 +0800 Subject: [PATCH 02/15] Code Optimization --- .../manager/engineplugin/io/utils/IOHelp.scala | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/utils/IOHelp.scala b/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/utils/IOHelp.scala index 8d84a1b0b0..8734dd7191 100644 --- a/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/utils/IOHelp.scala +++ b/linkis-engineconn-plugins/io_file/src/main/scala/org/apache/linkis/manager/engineplugin/io/utils/IOHelp.scala @@ -61,17 +61,21 @@ object IOHelp { writer.addRecord(ioRecord) writer.toString() } else if (method.params.length == 3) { - val parm1 = BigDecimal(method.params(1).toString).intValue() - val parm2 = BigDecimal(method.params(2).toString).intValue() - if (parm1 > 0) { - inputStream.skip(parm1) - } + val position = + if (method.params(1).toString.toDouble.toInt < 0) { + 0 + } else { + method.params(1).toString.toDouble.toInt + } val fetchSize = - if (parm2 > maxPageSize) { + if (method.params(2).toString.toDouble.toInt > maxPageSize) { maxPageSize.toInt } else { - parm2 + method.params(2).toString.toDouble.toInt } + if (position > 0) { + inputStream.skip(position) + } val bytes = new Array[Byte](fetchSize) val len = StorageUtils.readBytes(inputStream, bytes, fetchSize) val ioMetaData = new IOMetaData(0, len) From 39011c41205fb4c5aad0e5cd15afa6c6d6496334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Tue, 24 Dec 2024 11:27:27 +0800 Subject: [PATCH 03/15] Code Optimization --- .../org/apache/linkis/manager/am/restful/EngineRestfulApi.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java index 9041d3f4e4..3feca8c643 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java @@ -770,7 +770,8 @@ public Message killEngineByUpdateConfig(HttpServletRequest req, @RequestBody Jso && AMConfiguration.isUnAllowKilledEngineType(engineType)) { return Message.error("multi user engine does not support this feature(多用户引擎不支持此功能)"); } - if (engineType.equals(Configuration.GLOBAL_CONF_SYMBOL())) { + if (StringUtils.isNotBlank(engineType) + && engineType.equals(Configuration.GLOBAL_CONF_SYMBOL())) { Arrays.stream(AMConfiguration.UDF_KILL_ENGINE_TYPE.split(",")) .forEach( engine -> From f550f4978bcd6f3ae96f7585872c0f8b5b35521a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Tue, 24 Dec 2024 11:40:56 +0800 Subject: [PATCH 04/15] Code Optimization --- .../org/apache/linkis/manager/am/restful/EngineRestfulApi.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java index 3feca8c643..1e118e0ee7 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java @@ -770,8 +770,7 @@ public Message killEngineByUpdateConfig(HttpServletRequest req, @RequestBody Jso && AMConfiguration.isUnAllowKilledEngineType(engineType)) { return Message.error("multi user engine does not support this feature(多用户引擎不支持此功能)"); } - if (StringUtils.isNotBlank(engineType) - && engineType.equals(Configuration.GLOBAL_CONF_SYMBOL())) { + if (Configuration.GLOBAL_CONF_SYMBOL().equals(engineType)) { Arrays.stream(AMConfiguration.UDF_KILL_ENGINE_TYPE.split(",")) .forEach( engine -> From f8eeef3e5be86fda4e4ef71b441f9f8db7018067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Wed, 25 Dec 2024 17:59:38 +0800 Subject: [PATCH 05/15] fix miss code --- .../linkis/manager/am/converter/DefaultMetricsConverter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java index c7620157a1..132409c154 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java @@ -136,6 +136,7 @@ public AMNode fillMetricsToNode(AMNode amNode, NodeMetrics metrics) { amNode.setNodeHealthyInfo(parseHealthyInfo(metrics)); amNode.setNodeOverLoadInfo(parseOverLoadInfo(metrics)); amNode.setUpdateTime(metrics.getUpdateTime()); + amNode.setNodeDescription(metrics.getDescription()); return amNode; } } From 9f9c01813a7b3db935de48c334232efdf07e314d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Thu, 26 Dec 2024 10:46:21 +0800 Subject: [PATCH 06/15] New interface: Retrieve data source list based on type name --- .../restful/DataSourceCoreRestfulApi.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java index d6740b16d5..c6d99e4061 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java @@ -891,6 +891,44 @@ public Message encryptDatasourcePassword( "Fail to aes of data source[加密数据源密码失败]"); } + @ApiOperation( + value = "getDataSourceByTypeName", + notes = "get data source by datasource type name", + response = Message.class) + @RequestMapping(value = "/info-by-type", method = RequestMethod.GET) + public Message getDataSourceListByTypes( + HttpServletRequest request, + @RequestParam(value = "typeName", required = false) String typeName, + @RequestParam(value = "currentPage", required = false) Integer currentPage, + @RequestParam(value = "pageSize", required = false) Integer pageSize) { + return RestfulApiHelper.doAndResponse( + () -> { + String userName = ModuleUserUtils.getOperationUser(request, "getDataSourceByTypeName"); + if (AuthContext.isAdministrator(userName)) { + userName = null; + } + List queryList = new ArrayList<>(); + Message message = Message.ok(); + List dataSourceTypes = + dataSourceRelateService.getAllDataSourceTypes(request.getHeader("Content-Language")); + // 从dataSourceTypes过滤出typeName为typeName的数据源类型 + for (DataSourceType dataSourceType : dataSourceTypes) { + if (dataSourceType.getName().equals(typeName)) { + DataSourceVo dataSourceVo = + new DataSourceVo(null, Long.valueOf(dataSourceType.getId()), null, null); + dataSourceVo.setCurrentPage(null != currentPage ? currentPage : 1); + dataSourceVo.setPageSize(null != pageSize ? pageSize : 10); + dataSourceVo.setPermissionUser(userName); + PageInfo pageInfo = + dataSourceInfoService.queryDataSourceInfoPage(dataSourceVo); + queryList = pageInfo.getList(); + message.data("totalPage", pageInfo.getTotal()); + } + } + return message.data("queryList", queryList); + }, + "Fail to get all types of data source[获取数据源列表失败]"); + } /** * Inner method to insert data source * From 26248444667ce46715ae5df32372e22ade7051c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Thu, 26 Dec 2024 14:20:42 +0800 Subject: [PATCH 07/15] New interface: Retrieve data source list based on type name --- .../restful/DataSourceCoreRestfulApi.java | 70 ++++++++++----- .../GetDataSourceByTypeNameAction.scala | 85 +++++++++++++++++++ .../GetDataSourceKeyDefineByTypeAction.scala | 65 ++++++++++++++ .../GetDataSourceByTypeNameResult.scala | 44 ++++++++++ .../GetDataSourceKeyDefineByTypeResult.scala | 47 ++++++++++ 5 files changed, 292 insertions(+), 19 deletions(-) create mode 100644 linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceByTypeNameAction.scala create mode 100644 linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceKeyDefineByTypeAction.scala create mode 100644 linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceByTypeNameResult.scala create mode 100644 linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceKeyDefineByTypeResult.scala diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java index c6d99e4061..bbb43d94df 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java @@ -140,6 +140,37 @@ public Message getKeyDefinitionsByType( "Fail to get key definitions of data source type[查询数据源参数键值对失败]"); } + @ApiOperation( + value = "getKeyDefinitionsByTypeName", + notes = "get key definitions by typeName", + response = Message.class) + @ApiImplicitParams({@ApiImplicitParam(name = "typeName", required = true, dataType = "String")}) + @RequestMapping(value = "/key-define/{typeName}", method = RequestMethod.GET) + public Message getKeyDefinitionsByTypeName( + @PathVariable("typeName") String typeName, HttpServletRequest request) { + return RestfulApiHelper.doAndResponse( + () -> { + String userName = ModuleUserUtils.getOperationUser(request, "getKeyDefinitionsByType"); + List dataSourceTypes = + dataSourceRelateService.getAllDataSourceTypes(request.getHeader("Content-Language")); + DataSourceType targetDataSourceType = + dataSourceTypes.stream() + .filter(type -> type.getName().equals(typeName)) + .findFirst() + .orElse(null); + if (targetDataSourceType != null) { + List keyDefinitions = + dataSourceRelateService.getKeyDefinitionsByType( + Long.valueOf(targetDataSourceType.getId()), + request.getHeader("Content-Language")); + return Message.ok().data("keyDefine", keyDefinitions); + } else { + return Message.error("No data source type found with name: " + typeName); + } + }, + "Fail to get key definitions of data source type[查询数据源参数键值对失败]"); + } + @ApiOperation(value = "insertJsonInfo", notes = "insert json info", response = Message.class) @ApiOperationSupport(ignoreParameters = {"dataSource"}) @ApiImplicitParams({ @@ -898,34 +929,35 @@ public Message encryptDatasourcePassword( @RequestMapping(value = "/info-by-type", method = RequestMethod.GET) public Message getDataSourceListByTypes( HttpServletRequest request, - @RequestParam(value = "typeName", required = false) String typeName, - @RequestParam(value = "currentPage", required = false) Integer currentPage, - @RequestParam(value = "pageSize", required = false) Integer pageSize) { + @RequestParam String typeName, + @RequestParam(required = false, defaultValue = "1") Integer currentPage, + @RequestParam(required = false, defaultValue = "10") Integer pageSize) { return RestfulApiHelper.doAndResponse( () -> { String userName = ModuleUserUtils.getOperationUser(request, "getDataSourceByTypeName"); if (AuthContext.isAdministrator(userName)) { userName = null; } - List queryList = new ArrayList<>(); - Message message = Message.ok(); List dataSourceTypes = dataSourceRelateService.getAllDataSourceTypes(request.getHeader("Content-Language")); - // 从dataSourceTypes过滤出typeName为typeName的数据源类型 - for (DataSourceType dataSourceType : dataSourceTypes) { - if (dataSourceType.getName().equals(typeName)) { - DataSourceVo dataSourceVo = - new DataSourceVo(null, Long.valueOf(dataSourceType.getId()), null, null); - dataSourceVo.setCurrentPage(null != currentPage ? currentPage : 1); - dataSourceVo.setPageSize(null != pageSize ? pageSize : 10); - dataSourceVo.setPermissionUser(userName); - PageInfo pageInfo = - dataSourceInfoService.queryDataSourceInfoPage(dataSourceVo); - queryList = pageInfo.getList(); - message.data("totalPage", pageInfo.getTotal()); - } + DataSourceType targetDataSourceType = + dataSourceTypes.stream() + .filter(type -> type.getName().equals(typeName)) + .findFirst() + .orElse(null); + if (targetDataSourceType != null) { + DataSourceVo dataSourceVo = new DataSourceVo(); + dataSourceVo.setDataSourceTypeId(Long.valueOf(targetDataSourceType.getId())); + dataSourceVo.setPermissionUser(userName); + dataSourceVo.setCurrentPage(currentPage); + dataSourceVo.setPageSize(pageSize); + PageInfo pageInfo = + dataSourceInfoService.queryDataSourceInfoPage(dataSourceVo); + List queryList = pageInfo.getList(); + return Message.ok().data("queryList", queryList).data("totalPage", pageInfo.getTotal()); + } else { + return Message.error("No data source type found with name: " + typeName); } - return message.data("queryList", queryList); }, "Fail to get all types of data source[获取数据源列表失败]"); } diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceByTypeNameAction.scala b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceByTypeNameAction.scala new file mode 100644 index 0000000000..25022b2252 --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceByTypeNameAction.scala @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.datasource.client.request + +import org.apache.linkis.datasource.client.config.DatasourceClientConfig.DATA_SOURCE_SERVICE_MODULE +import org.apache.linkis.datasource.client.errorcode.DatasourceClientErrorCodeSummary._ +import org.apache.linkis.datasource.client.exception.DataSourceClientBuilderException +import org.apache.linkis.datasource.client.request.QueryDataSourceAction.Builder +import org.apache.linkis.httpclient.request.GetAction + +class GetDataSourceByTypeNameAction extends GetAction with DataSourceAction { + private var user: String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = + Array(DATA_SOURCE_SERVICE_MODULE.getValue, "info-by-type") + +} + +object GetDataSourceByTypeNameAction { + def builder(): Builder = new Builder + + class Builder private[GetDataSourceByTypeNameAction] () { + private var user: String = _ + private var typeName: String = _ + private var currentPage: Integer = _ + private var pageSize: Integer = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setTypeName(typeName: String): Builder = { + this.typeName = typeName + this + } + + def setCurrentPage(currentPage: Integer): Builder = { + this.currentPage = currentPage + this + } + + def setPageSize(pageSize: Integer): Builder = { + this.pageSize = pageSize + this + } + + def build(): GetDataSourceByTypeNameAction = { + if (user == null) throw new DataSourceClientBuilderException(USER_NEEDED.getErrorDesc) + val action = new GetDataSourceByTypeNameAction + if (typeName != null) { + action.setParameter("typeName", typeName) + } + if (currentPage != null) { + action.setParameter("currentPage", currentPage) + } + if (pageSize != null) { + action.setParameter("pageSize", pageSize) + } + action.user = user + action + } + + } + +} diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceKeyDefineByTypeAction.scala b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceKeyDefineByTypeAction.scala new file mode 100644 index 0000000000..854351a5c1 --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceKeyDefineByTypeAction.scala @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.datasource.client.request + +import org.apache.linkis.datasource.client.config.DatasourceClientConfig.DATA_SOURCE_SERVICE_MODULE +import org.apache.linkis.datasource.client.errorcode.DatasourceClientErrorCodeSummary._ +import org.apache.linkis.datasource.client.exception.DataSourceClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class GetDataSourceKeyDefineByTypeAction extends GetAction with DataSourceAction { + private var user: String = _ + private var typeName: String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = + Array(DATA_SOURCE_SERVICE_MODULE.getValue, "key-define", typeName) + +} + +object GetDataSourceKeyDefineByTypeAction { + def builder(): Builder = new Builder + + class Builder private[GetDataSourceKeyDefineByTypeAction] () { + private var user: String = _ + private var typeName: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setTypeName(typeName: String): Builder = { + this.typeName = typeName + this + } + + def build(): GetDataSourceKeyDefineByTypeAction = { + if (user == null) throw new DataSourceClientBuilderException(USER_NEEDED.getErrorDesc) + val action = new GetDataSourceKeyDefineByTypeAction + action.user = user + action.typeName = typeName + action + } + + } + +} diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceByTypeNameResult.scala b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceByTypeNameResult.scala new file mode 100644 index 0000000000..5165664301 --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceByTypeNameResult.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.datasource.client.response + +import org.apache.linkis.datasourcemanager.common.domain.DataSource +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-source-manager/info-by-type") +class GetDataSourceByTypeNameResult extends DWSResult { + @BeanProperty var queryList: util.List[java.util.Map[String, Any]] = _ + @BeanProperty var totalPage: Int = _ + + def getAllDataSource: util.List[DataSource] = { + import scala.collection.JavaConverters._ + queryList.asScala + .map(x => { + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[DataSource]) + }) + .asJava + } + +} diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceKeyDefineByTypeResult.scala b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceKeyDefineByTypeResult.scala new file mode 100644 index 0000000000..16e3b2c063 --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceKeyDefineByTypeResult.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.datasource.client.response + +import org.apache.linkis.datasourcemanager.common.domain.{ + DataSourceParamKeyDefinition, + DatasourceVersion +} +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-source-manager/key-define/(\\S+)") +class GetDataSourceKeyDefineByTypeResult extends DWSResult { + + @BeanProperty var keyDefine: util.List[java.util.Map[String, Any]] = _ + + def getDatasourceVersion: util.List[DataSourceParamKeyDefinition] = { + import scala.collection.JavaConverters._ + keyDefine.asScala + .map(x => { + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[DataSourceParamKeyDefinition]) + }) + .asJava + } + +} From 423b48d48f35ab3f8243c85c64c30d99a5c17b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Thu, 26 Dec 2024 19:03:12 +0800 Subject: [PATCH 08/15] bug fix --- .../java/org/apache/linkis/manager/am/conf/AMConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java index 903c58ada7..c1d96fb931 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java @@ -101,7 +101,7 @@ public class AMConfiguration { CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python"); public static final CommonVars UNALLOW_BATCH_KILL_ENGINE_TYPES = - CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "trino,appconn,io_file,jdbc"); + CommonVars.apply("wds.linkis.unallow.batch.kill.engine.types", "trino,appconn,io_file,jdbc"); public static final CommonVars MULTI_USER_ENGINE_USER = CommonVars.apply("wds.linkis.multi.user.engine.user", getDefaultMultiEngineUser()); public static final String UDF_KILL_ENGINE_TYPE = From 88add695620e8e6388d5ba90d0e9535571151b27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Mon, 30 Dec 2024 14:42:20 +0800 Subject: [PATCH 09/15] bug fix --- .../apache/linkis/metadata/service/impl/MdqServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/MdqServiceImpl.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/MdqServiceImpl.java index 25afc1c44a..dea17cca95 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/MdqServiceImpl.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/MdqServiceImpl.java @@ -351,7 +351,7 @@ private MdqTablePartitionStatisticInfoVO create(String path) throws IOException } private Date getTableModificationTime(String tableLocation) throws IOException { - if (StringUtils.isNotBlank(tableLocation)) { + if (StringUtils.isNotBlank(tableLocation) && getRootHdfs().exists(new Path(tableLocation))) { FileStatus tableFile = getFileStatus(tableLocation); return new Date(tableFile.getModificationTime()); } @@ -360,7 +360,7 @@ private Date getTableModificationTime(String tableLocation) throws IOException { private int getPartitionsNum(String tableLocation) throws IOException { int partitionsNum = 0; - if (StringUtils.isNotBlank(tableLocation)) { + if (StringUtils.isNotBlank(tableLocation) && getRootHdfs().exists(new Path(tableLocation))) { FileStatus tableFile = getFileStatus(tableLocation); partitionsNum = getRootHdfs().listStatus(tableFile.getPath()).length; } From b8f08deb9dd48a8cfdd7dbc56459543644ea3312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Mon, 6 Jan 2025 15:21:26 +0800 Subject: [PATCH 10/15] code review fix --- .../apache/linkis/common/utils/SHAUtils.java | 27 +++++++++++++++++++ .../linkis/monitor/until/ThreadUtils.java | 8 ++++++ .../analyze/JobHistoryAnalyzeRule.scala | 5 ++-- .../HiveMetaWithPermissionServiceImpl.java | 5 +++- .../restful/api/QueryRestfulApi.java | 4 +-- .../jobhistory/transitional/TaskStatus.java | 10 +++++++ .../linkis/jobhistory/util/Constants.java | 4 --- 7 files changed, 53 insertions(+), 10 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SHAUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SHAUtils.java index 8122de8558..cbb9d83833 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SHAUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SHAUtils.java @@ -19,9 +19,14 @@ import org.apache.linkis.common.conf.CommonVars; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Map; public class SHAUtils { @@ -66,4 +71,26 @@ public static String bytes2Hex(byte[] bts) { } return des; } + + public static void main(String[] args) throws IOException { + if (StringUtils.isBlank(args[0])) { + throw new LinkageError("Invalid applicationId cannot be empty"); + } + Map parms = new HashMap<>(); + String timestampStr = String.valueOf(System.currentTimeMillis()); + parms.put("applicationId", args[0]); + parms.put("app_id", args[1]); + parms.put("timestamp", timestampStr); + parms.put("nonce", DOCTOR_NONCE); + String token = args[2]; + if (StringUtils.isNotBlank(token)) { + String signature = + Encrypt( + Encrypt(parms.get("app_id") + DOCTOR_NONCE + System.currentTimeMillis(), null) + + token, + null); + parms.put("signature", signature); + } + System.out.println(parms); + } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/ThreadUtils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/ThreadUtils.java index 15a2626379..54c81b948d 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/ThreadUtils.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/ThreadUtils.java @@ -20,6 +20,7 @@ import org.apache.linkis.common.utils.Utils; import org.apache.linkis.monitor.config.MonitorConfig; import org.apache.linkis.monitor.constants.Constants; +import org.apache.linkis.monitor.jobhistory.entity.JobHistory; import org.apache.linkis.monitor.utils.alert.AlertDesc; import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; import org.apache.linkis.monitor.utils.alert.ims.PooledImsAlertUtils; @@ -42,6 +43,9 @@ public class ThreadUtils extends ApplicationContextEvent { public static ExecutionContextExecutorService executors = Utils.newCachedExecutionContext(5, "alert-pool-thread-", false); + public static ExecutionContextExecutorService executors_analyze = + Utils.newCachedExecutionContext(50, "analyze-pool-thread-", false); + public ThreadUtils(ApplicationContext source) { super(source); } @@ -64,4 +68,8 @@ public static String run(List cmdList, String shellName) { } return msg; } + public static void analyzeRun(JobHistory jobHistory){ + FutureTask future = new FutureTask(() -> HttpsUntils.analyzeJob(jobHistory), -1); + executors_analyze.submit(future); + } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/analyze/JobHistoryAnalyzeRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/analyze/JobHistoryAnalyzeRule.scala index 2a95de8571..aec128ffab 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/analyze/JobHistoryAnalyzeRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/analyze/JobHistoryAnalyzeRule.scala @@ -17,8 +17,7 @@ package org.apache.linkis.monitor.jobhistory.analyze -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.monitor.config.MonitorConfig +import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.monitor.constants.Constants import org.apache.linkis.monitor.core.ob.Observer import org.apache.linkis.monitor.core.pac.{AbstractScanRule, ScannedData} @@ -50,7 +49,7 @@ class JobHistoryAnalyzeRule(hitObserver: Observer) val jobStatus = jobHistory.getStatus.toUpperCase() if (Constants.FINISHED_JOB_STATUS.contains(jobStatus) && jobStatus.equals("FAILED")) { // 执行任务分析 - HttpsUntils.analyzeJob(jobHistory) + ThreadUtils.analyzeRun(jobHistory) } case _ => } diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/HiveMetaWithPermissionServiceImpl.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/HiveMetaWithPermissionServiceImpl.java index 88a4d93b6e..365e5bfde6 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/HiveMetaWithPermissionServiceImpl.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/HiveMetaWithPermissionServiceImpl.java @@ -32,6 +32,7 @@ import org.springframework.stereotype.Service; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -104,7 +105,9 @@ public List> getTablesByDbNameAndOptionalUserName( List> hiveTables = hiveMetaDao.getTablesByDbNameAndUserAndRolesFromDbPrvs(queryParam); hiveTables.addAll(hiveMetaDao.getTablesByDbNameAndUserAndRolesFromTblPrvs(queryParam)); - return hiveTables.stream().distinct().collect(Collectors.toList()); + return hiveTables.stream() + .sorted(Comparator.comparing(hiveTable -> (String) hiveTable.get("NAME"))) + .collect(Collectors.toList()); } else { log.info("user {} to getTablesByDbName no permission control", queryParam.getUserName()); return hiveMetaDao.getTablesByDbName(queryParam); diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java index f40d028726..6389338b70 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java @@ -839,7 +839,7 @@ public Message queryFailedTaskDiagnosis( jobDiagnosis.setDiagnosisContent(diagnosisMsg); jobDiagnosis.setCreatedTime(new Date()); jobDiagnosis.setUpdatedDate(new Date()); - if (Constants.FINISHED_JOB_STATUS.contains(jobStatus)) { + if (TaskStatus.isComplete(TaskStatus.valueOf(jobStatus))) { jobDiagnosis.setOnlyRead("1"); } jobHistoryDiagnosisService.insert(jobDiagnosis); @@ -852,7 +852,7 @@ public Message queryFailedTaskDiagnosis( jobDiagnosis.setDiagnosisContent(diagnosisMsg); jobDiagnosis.setUpdatedDate(new Date()); jobDiagnosis.setDiagnosisContent(diagnosisMsg); - if (Constants.FINISHED_JOB_STATUS.contains(jobStatus)) { + if (TaskStatus.isComplete(TaskStatus.valueOf(jobStatus))) { jobDiagnosis.setOnlyRead("1"); } jobHistoryDiagnosisService.update(jobDiagnosis); diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/transitional/TaskStatus.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/transitional/TaskStatus.java index 191ba4ab51..11b19b88e4 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/transitional/TaskStatus.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/transitional/TaskStatus.java @@ -40,4 +40,14 @@ public static boolean isComplete(TaskStatus taskStatus) { return false; } } + public static boolean isUnComplete(TaskStatus taskStatus) { + if (taskStatus == Inited + || taskStatus == WaitForRetry + || taskStatus == Scheduled + || taskStatus == Running) { + return true; + } else { + return false; + } + } } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/Constants.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/Constants.java index 3986fef636..e18a7c49c4 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/Constants.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/Constants.java @@ -20,8 +20,4 @@ public class Constants { public static final String APPLICATION_NAME = "linkis-ps-publicservice"; - public static final String UNFINISHED_JOB_STATUS = - "Inited,WaitForRetry,Scheduled,Running".toUpperCase(); - - public static final String FINISHED_JOB_STATUS = "Succeed,Failed,Cancelled,Timeout".toUpperCase(); } From 8eef9b193e062f3af3ab308fbe092fb3b8767420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Mon, 6 Jan 2025 16:08:38 +0800 Subject: [PATCH 11/15] code review fix --- .../restful/DataSourceCoreRestfulApi.java | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java index bbb43d94df..2e8beaac2c 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java @@ -891,6 +891,63 @@ public Message queryDataSource( "Fail to query page of data source[查询数据源失败]"); } + @ApiOperation( + value = "queryDataSourceWithConnectParms", + notes = "query datasource", + response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "system", dataType = "String"), + @ApiImplicitParam(name = "name", dataType = "Long"), + @ApiImplicitParam(name = "typeId", dataType = "Long"), + @ApiImplicitParam(name = "identifies", dataType = "String"), + @ApiImplicitParam(name = "currentPage", dataType = "Integer"), + @ApiImplicitParam(name = "pageSize", dataType = "Integer") + }) + @RequestMapping(value = "/info/connect-params", method = RequestMethod.GET) + public Message queryDataSourceWithConnectParms( + @RequestParam(value = "system", required = false) String createSystem, + @RequestParam(value = "name", required = false) String dataSourceName, + @RequestParam(value = "typeId", required = false) Long dataSourceTypeId, + @RequestParam(value = "identifies", required = false) String identifies, + @RequestParam(value = "currentPage", required = false) Integer currentPage, + @RequestParam(value = "pageSize", required = false) Integer pageSize, + HttpServletRequest request) { + return RestfulApiHelper.doAndResponse( + () -> { + String permissionUser = ModuleUserUtils.getOperationUser(request, "queryDataSourceWithConnectParms"); + + DataSourceVo dataSourceVo = + new DataSourceVo(dataSourceName, dataSourceTypeId, identifies, createSystem); + dataSourceVo.setCurrentPage(null != currentPage ? currentPage : 1); + dataSourceVo.setPageSize(null != pageSize ? pageSize : 10); + + if (AuthContext.isAdministrator(permissionUser)) { + permissionUser = null; + } + dataSourceVo.setPermissionUser(permissionUser); + PageInfo pageInfo = + dataSourceInfoService.queryDataSourceInfoPage(dataSourceVo); + List queryList = pageInfo.getList(); + for (DataSource dataSource : queryList) { + DataSource dataSourceConnect = + dataSourceInfoService.getDataSourceInfoForConnect(dataSource.getDataSourceName()); + if (dataSourceConnect == null) { + return Message.error("No Exists The DataSource [不存在该数据源]"); + } + Map connectParams = dataSourceConnect.getConnectParams(); + List keyDefinitionList = + dataSourceRelateService.getKeyDefinitionsByType( + dataSourceConnect.getDataSourceTypeId()); + if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams); + } + dataSource.setConnectParams(connectParams); + } + return Message.ok().data("queryList", queryList).data("totalPage", pageInfo.getTotal()); + }, + "Fail to query page of data source[查询数据源失败]"); + } + @ApiOperation( value = "encryptDatasourcePassword", notes = "encrypt datasource password", From bff0228218ea75afba3231813e8f0f3547ee0962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Mon, 6 Jan 2025 16:36:56 +0800 Subject: [PATCH 12/15] code review fix --- .../restful/DataSourceCoreRestfulApi.java | 3 +- .../restful/api/QueryRestfulApi.java | 35 ------------------- 2 files changed, 2 insertions(+), 36 deletions(-) diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java index 2e8beaac2c..5cc5ebf68a 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java @@ -914,7 +914,7 @@ public Message queryDataSourceWithConnectParms( HttpServletRequest request) { return RestfulApiHelper.doAndResponse( () -> { - String permissionUser = ModuleUserUtils.getOperationUser(request, "queryDataSourceWithConnectParms"); + String permissionUser = ModuleUserUtils.getOperationUser(request, "queryDataSource"); DataSourceVo dataSourceVo = new DataSourceVo(dataSourceName, dataSourceTypeId, identifies, createSystem); @@ -941,6 +941,7 @@ public Message queryDataSourceWithConnectParms( if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams); } + connectParams.remove(AESUtils.PASSWORD); dataSource.setConnectParams(connectParams); } return Message.ok().data("queryList", queryList).data("totalPage", pageInfo.getTotal()); diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java index 6389338b70..8f7ab8b206 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java @@ -754,41 +754,6 @@ private List getJobhistoryList( return queryTasks; } - @ApiOperation(value = "get-doctor-signature", notes = "get signature", response = Message.class) - @ApiImplicitParams({ - @ApiImplicitParam(name = "applicationId", required = false, dataType = "String", value = "user") - }) - @RequestMapping(path = "/get-doctor-signature", method = RequestMethod.GET) - public Message getUserKeyValue( - HttpServletRequest req, - @RequestParam(value = "applicationId", required = false) String applicationId) - throws UnsupportedEncodingException { - if (StringUtils.isBlank(applicationId)) { - return Message.error("Invalid applicationId cannot be empty"); - } - Map parms = new HashMap<>(); - String timestampStr = String.valueOf(System.currentTimeMillis()); - parms.put("applicationId", applicationId); - parms.put("app_id", Configuration.LINKIS_SYS_NAME().getValue()); - parms.put("timestamp", timestampStr); - parms.put("nonce", SHAUtils.DOCTOR_NONCE); - // doctor提供的token - String token = SHAUtils.DOCTOR_TOKEN.getValue(); - if (StringUtils.isNotBlank(token)) { - String signature = - SHAUtils.Encrypt( - SHAUtils.Encrypt( - parms.get("app_id") + SHAUtils.DOCTOR_NONCE + System.currentTimeMillis(), - null) - + token, - null); - parms.put("signature", signature); - return Message.ok().data("doctor", parms); - } else { - return Message.error("Doctor token cannot be empty"); - } - } - @ApiOperation( value = "diagnosis-query", notes = "query failed task diagnosis msg", From d7564890f3592507e6b19cbf46529fd72a94906d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Tue, 7 Jan 2025 14:29:29 +0800 Subject: [PATCH 13/15] code review fix --- .../linkis/monitor/until/ThreadUtils.java | 5 +- .../restful/api/DataSourceRestfulApi.java | 16 +++++ .../remote/DataSourceRestfulRemote.java | 3 + .../metadata/service/DataSourceService.java | 4 ++ .../service/impl/DataSourceServiceImpl.java | 60 +++++++++++++++++-- .../HiveMetaWithPermissionServiceImpl.java | 4 +- .../restful/api/QueryRestfulApi.java | 5 +- .../jobhistory/transitional/TaskStatus.java | 7 ++- .../linkis/jobhistory/util/Constants.java | 1 - .../apache/linkis/udf/api/UDFRestfulApi.java | 3 +- 10 files changed, 91 insertions(+), 17 deletions(-) diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/ThreadUtils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/ThreadUtils.java index 54c81b948d..5e4133aa90 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/ThreadUtils.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/ThreadUtils.java @@ -44,7 +44,7 @@ public class ThreadUtils extends ApplicationContextEvent { Utils.newCachedExecutionContext(5, "alert-pool-thread-", false); public static ExecutionContextExecutorService executors_analyze = - Utils.newCachedExecutionContext(50, "analyze-pool-thread-", false); + Utils.newCachedExecutionContext(50, "analyze-pool-thread-", false); public ThreadUtils(ApplicationContext source) { super(source); @@ -68,7 +68,8 @@ public static String run(List cmdList, String shellName) { } return msg; } - public static void analyzeRun(JobHistory jobHistory){ + + public static void analyzeRun(JobHistory jobHistory) { FutureTask future = new FutureTask(() -> HttpsUntils.analyzeJob(jobHistory), -1); executors_analyze.submit(future); } diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/restful/api/DataSourceRestfulApi.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/restful/api/DataSourceRestfulApi.java index cece989f2d..7b059e87e0 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/restful/api/DataSourceRestfulApi.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/restful/api/DataSourceRestfulApi.java @@ -131,6 +131,22 @@ public Message queryDbsWithTables(HttpServletRequest req) { } } + @ApiOperation( + value = "queryDbsWithTablesgetByAccessTime", + notes = "query dbs with tables order by getByAccessTime", + response = Message.class) + @RequestMapping(path = "getByAccessTime", method = RequestMethod.GET) + public Message queryDbsWithTablesgetByAccessTime(HttpServletRequest req) { + String userName = ModuleUserUtils.getOperationUser(req, "get all db and tables"); + try { + JsonNode dbs = dataSourceService.getDbsWithTablesAndLastAccessAt(userName); + return Message.ok("").data("dbs", dbs); + } catch (Exception e) { + logger.error("Failed to queryDbsWithTables", e); + return Message.error("Failed to queryDbsWithTables", e); + } + } + @ApiOperation(value = "queryTables", notes = "query tables", response = Message.class) @ApiImplicitParams({ @ApiImplicitParam(name = "database", dataType = "String", value = "database") diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/restful/remote/DataSourceRestfulRemote.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/restful/remote/DataSourceRestfulRemote.java index 06db07c46a..da4801e15a 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/restful/remote/DataSourceRestfulRemote.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/restful/remote/DataSourceRestfulRemote.java @@ -35,6 +35,9 @@ public interface DataSourceRestfulRemote { @GetMapping("/api/datasource/all") public Message queryDbsWithTables(HttpServletRequest req); + @GetMapping("/api/datasource/getByAccessTime") + public Message queryDbsWithTablesgetByAccessTime(HttpServletRequest req); + @GetMapping("/api/datasource/tables") public Message queryTables(@RequestParam("database") String database, HttpServletRequest req); diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/DataSourceService.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/DataSourceService.java index 46cf626419..ed2d64cc6a 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/DataSourceService.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/DataSourceService.java @@ -29,8 +29,12 @@ public interface DataSourceService { JsonNode getDbsWithTables(String userName) throws Exception; + JsonNode getDbsWithTablesAndLastAccessAt(String userName) throws Exception; + JsonNode queryTables(MetadataQueryParam queryParam); + JsonNode queryTablesWithLastAccessAt(MetadataQueryParam queryParam); + JsonNode queryTableMeta(MetadataQueryParam queryParam); JsonNode queryTableMetaBySDID(MetadataQueryParam queryParam); diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/DataSourceServiceImpl.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/DataSourceServiceImpl.java index b179a66560..5dcac25dc5 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/DataSourceServiceImpl.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/DataSourceServiceImpl.java @@ -41,10 +41,7 @@ import org.springframework.stereotype.Service; import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -102,6 +99,33 @@ public JsonNode getDbsWithTables(String userName) { return dbNodes; } + @DataSource(name = DSEnum.FIRST_DATA_SOURCE) + @Override + public JsonNode getDbsWithTablesAndLastAccessAt(String userName) { + ArrayNode dbNodes = jsonMapper.createArrayNode(); + List dbs = hiveMetaWithPermissionService.getDbsOptionalUserName(userName, null); + MetadataQueryParam queryParam = MetadataQueryParam.of(userName); + int count = 0; + for (String db : dbs) { + if (StringUtils.isBlank(db) || db.contains(dbKeyword)) { + logger.info("db will be filter: " + db); + continue; + } + JsonNode jsonNode = queryTablesWithLastAccessAt(queryParam); + count += jsonNode.size(); + if (count < 100000) { + queryParam.setDbName(db); + ObjectNode dbNode = jsonMapper.createObjectNode(); + dbNode.put("databaseName", db); + dbNode.put("tables", jsonNode); + dbNodes.add(dbNode); + } else { + break; + } + } + return dbNodes; + } + @DataSource(name = DSEnum.FIRST_DATA_SOURCE) @Override public JsonNode queryTables(MetadataQueryParam queryParam) { @@ -127,6 +151,34 @@ public JsonNode queryTables(MetadataQueryParam queryParam) { return tables; } + @DataSource(name = DSEnum.FIRST_DATA_SOURCE) + @Override + public JsonNode queryTablesWithLastAccessAt(MetadataQueryParam queryParam) { + List> listTables; + try { + listTables = hiveMetaWithPermissionService.getTablesByDbNameAndOptionalUserName(queryParam); + } catch (Throwable e) { + logger.error("Failed to list Tables:", e); + throw new RuntimeException(e); + } + long threeMonthsAgo = System.currentTimeMillis() - (3L * 30 * 24 * 60 * 60 * 1000); + ArrayNode tables = jsonMapper.createArrayNode(); + listTables.stream() + .filter(table -> ((Integer) table.get("LAST_ACCESS_TIME")).longValue() > threeMonthsAgo) + .forEach( + table -> { + ObjectNode tableNode = jsonMapper.createObjectNode(); + tableNode.put("tableName", (String) table.get("NAME")); + tableNode.put("isView", table.get("TYPE").equals("VIRTUAL_VIEW")); + tableNode.put("databaseName", queryParam.getDbName()); + tableNode.put("createdBy", (String) table.get("OWNER")); + tableNode.put("createdAt", (Integer) table.get("CREATE_TIME")); + tableNode.put("lastAccessAt", (Integer) table.get("LAST_ACCESS_TIME")); + tables.add(tableNode); + }); + return tables; + } + @DataSource(name = DSEnum.FIRST_DATA_SOURCE) @Override public JsonNode queryTableMeta(MetadataQueryParam queryParam) { diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/HiveMetaWithPermissionServiceImpl.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/HiveMetaWithPermissionServiceImpl.java index 365e5bfde6..c56b07a1d6 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/HiveMetaWithPermissionServiceImpl.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/HiveMetaWithPermissionServiceImpl.java @@ -106,8 +106,8 @@ public List> getTablesByDbNameAndOptionalUserName( hiveMetaDao.getTablesByDbNameAndUserAndRolesFromDbPrvs(queryParam); hiveTables.addAll(hiveMetaDao.getTablesByDbNameAndUserAndRolesFromTblPrvs(queryParam)); return hiveTables.stream() - .sorted(Comparator.comparing(hiveTable -> (String) hiveTable.get("NAME"))) - .collect(Collectors.toList()); + .sorted(Comparator.comparing(hiveTable -> (String) hiveTable.get("NAME"))) + .collect(Collectors.toList()); } else { log.info("user {} to getTablesByDbName no permission control", queryParam.getUserName()); return hiveMetaDao.getTablesByDbName(queryParam); diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java index 8f7ab8b206..76f4373dd7 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java @@ -19,7 +19,6 @@ import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.common.exception.LinkisCommonErrorException; -import org.apache.linkis.common.utils.SHAUtils; import org.apache.linkis.governance.common.constant.job.JobRequestConstants; import org.apache.linkis.governance.common.entity.job.QueryException; import org.apache.linkis.jobhistory.cache.impl.DefaultQueryCacheManager; @@ -29,7 +28,6 @@ import org.apache.linkis.jobhistory.service.JobHistoryDiagnosisService; import org.apache.linkis.jobhistory.service.JobHistoryQueryService; import org.apache.linkis.jobhistory.transitional.TaskStatus; -import org.apache.linkis.jobhistory.util.Constants; import org.apache.linkis.jobhistory.util.JobhistoryUtils; import org.apache.linkis.jobhistory.util.QueryUtils; import org.apache.linkis.protocol.constants.TaskConstant; @@ -53,7 +51,6 @@ import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -795,7 +792,7 @@ public Message queryFailedTaskDiagnosis( } String diagnosisMsg = ""; if (jobHistory != null) { - String jobStatus = jobHistory.getStatus().toUpperCase(); + String jobStatus = jobHistory.getStatus(); JobDiagnosis jobDiagnosis = jobHistoryDiagnosisService.selectByJobId(Long.valueOf(taskID)); if (null == jobDiagnosis) { diagnosisMsg = JobhistoryUtils.getDiagnosisMsg(taskID); diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/transitional/TaskStatus.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/transitional/TaskStatus.java index 11b19b88e4..0c68f529f5 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/transitional/TaskStatus.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/transitional/TaskStatus.java @@ -40,11 +40,12 @@ public static boolean isComplete(TaskStatus taskStatus) { return false; } } + public static boolean isUnComplete(TaskStatus taskStatus) { if (taskStatus == Inited - || taskStatus == WaitForRetry - || taskStatus == Scheduled - || taskStatus == Running) { + || taskStatus == WaitForRetry + || taskStatus == Scheduled + || taskStatus == Running) { return true; } else { return false; diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/Constants.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/Constants.java index e18a7c49c4..11ba76c688 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/Constants.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/util/Constants.java @@ -19,5 +19,4 @@ public class Constants { public static final String APPLICATION_NAME = "linkis-ps-publicservice"; - } diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index 55b92034e2..a40d64a406 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -1445,7 +1445,8 @@ public Message pythonUpload( value = "path", example = "file:///test-dir/test-sub-dir/test1012_01.py") @RequestMapping(path = "/get-register-functions", method = RequestMethod.GET) - public Message getRegisterFunctions(HttpServletRequest req, @RequestParam("path") String path) throws IOException { + public Message getRegisterFunctions(HttpServletRequest req, @RequestParam("path") String path) + throws IOException { if (StringUtils.endsWithIgnoreCase(path, Constants.FILE_EXTENSION_PY) || StringUtils.endsWithIgnoreCase(path, Constants.FILE_EXTENSION_SCALA)) { if (StringUtils.startsWithIgnoreCase(path, StorageUtils$.MODULE$.FILE_SCHEMA())) { From d86c85e731cbc81779b943288c86156272d14f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Wed, 8 Jan 2025 11:40:15 +0800 Subject: [PATCH 14/15] Security work order repair --- .../linkis/jobhistory/restful/api/QueryRestfulApi.java | 3 +++ .../org/apache/linkis/jobhistory/util/QueryUtils.scala | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java index 76f4373dd7..a9a3c09b48 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java @@ -762,6 +762,9 @@ public Message queryFailedTaskDiagnosis( if (StringUtils.isBlank(taskID)) { return Message.error("Invalid jobId cannot be empty"); } + if (!QueryUtils.checkNumberValid(taskID)) { + throw new LinkisCommonErrorException(21304, "Invalid taskID : " + taskID); + } JobHistory jobHistory = null; boolean isAdmin = Configuration.isJobHistoryAdmin(username) || Configuration.isAdmin(username); boolean isDepartmentAdmin = Configuration.isDepartmentAdmin(username); diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala index 582183d07f..1e8d9370f2 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala @@ -50,8 +50,10 @@ object QueryUtils extends Logging { private val LENGTH_SPLIT = "#" private val NAME_REGEX = "^[a-zA-Z\\-\\d_\\.]+$" private val INSTANCE_NAME_REGEX = "^[a-zA-Z\\-\\d_\\.:]+$" + private val NUMBER_REGEX = "^[0-9]+$" private val nameRegexPattern = Pattern.compile(NAME_REGEX) private val instanceNameRegexPattern = Pattern.compile(INSTANCE_NAME_REGEX) + private val numberRegexPattern = Pattern.compile(NUMBER_REGEX) private val dateFormatLocal = new ThreadLocal[SimpleDateFormat]() { override protected def initialValue = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") @@ -173,4 +175,8 @@ object QueryUtils extends Logging { instanceNameRegexPattern.matcher(param).find() } + def checkNumberValid(param: String): Boolean = { + numberRegexPattern.matcher(param).find() + } + } From 8c74f79a3c5b7a33717a2ae9932dbd88956f59c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Wed, 8 Jan 2025 15:34:53 +0800 Subject: [PATCH 15/15] revent decimal fix --- .../org/apache/linkis/storage/domain/DataType.scala | 13 ++++--------- .../linkis/storage/excel/StorageExcelWriter.scala | 8 ++++---- .../linkis/ujes/client/utils/UJESClientUtils.scala | 4 +--- .../engineplugin/jdbc/executor/JDBCHelper.java | 2 +- .../execution/datasources/csv/DolphinToSpark.scala | 2 +- 5 files changed, 11 insertions(+), 18 deletions(-) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala index a603e08775..036cd8bfa2 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala @@ -62,7 +62,7 @@ object DataType extends Logging { case "date" => DateType case "timestamp" => TimestampType case "binary" => BinaryType - case "decimal" | DECIMAL_REGEX() => DecimalType(dataType, 3) + case "decimal" | DECIMAL_REGEX() => DecimalType case ARRAY_REGEX() => ArrayType case MAP_REGEX() => MapType case LIST_REGEX() => ListType @@ -89,8 +89,7 @@ object DataType extends Logging { case LongType | BigIntType => if (isNumberNull(newValue)) null else newValue.toLong case FloatType => if (isNumberNull(newValue)) null else newValue.toFloat case DoubleType => if (isNumberNull(newValue)) null else newValue.toDouble - case DecimalType(_, _) => - if (isNumberNull(newValue)) null else new JavaBigDecimal(newValue) + case DecimalType => if (isNumberNull(newValue)) null else new JavaBigDecimal(newValue) case DateType => if (isNumberNull(newValue)) null else Date.valueOf(newValue) case TimestampType => if (isNumberNull(newValue)) null else Timestamp.valueOf(newValue).toString.stripSuffix(".0") @@ -146,16 +145,12 @@ case object VarcharType extends DataType("varchar", 12) case object DateType extends DataType("date", 91) case object TimestampType extends DataType("timestamp", 93) case object BinaryType extends DataType("binary", -2) +case object DecimalType extends DataType("decimal", 3) case object ArrayType extends DataType("array", 2003) case object MapType extends DataType("map", 2000) case object ListType extends DataType("list", 2001) case object StructType extends DataType("struct", 2002) - -case class DecimalType(override val typeName: String, override val javaSQLType: Int) - extends DataType(typeName, javaSQLType) - -case class BigDecimalType(override val typeName: String, override val javaSQLType: Int) - extends DataType(typeName, javaSQLType) +case object BigDecimalType extends DataType("bigdecimal", 3) case class Column(columnName: String, dataType: DataType, comment: String) { diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala index 3740dc1aaf..9ea83130df 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala @@ -97,8 +97,8 @@ class StorageExcelWriter( case VarcharType => style.setDataFormat(format.getFormat("@")) case DateType => style.setDataFormat(format.getFormat("m/d/yy h:mm")) case TimestampType => style.setDataFormat(format.getFormat("m/d/yy h:mm")) - case DecimalType(_, _) => style.setDataFormat(format.getFormat("#.000000000")) - case BigDecimalType(_, _) => style.setDataFormat(format.getFormat("#.000000000")) + case DecimalType => style.setDataFormat(format.getFormat("#.000000000")) + case BigDecimalType => style.setDataFormat(format.getFormat("#.000000000")) case _ => style.setDataFormat(format.getFormat("@")) } } @@ -171,10 +171,10 @@ class StorageExcelWriter( case VarcharType => cell.setCellValue(DataType.valueToString(elem)) case DateType => cell.setCellValue(getDate(elem)) case TimestampType => cell.setCellValue(getDate(elem)) - case DecimalType(_, _) => + case DecimalType => doubleCheck(DataType.valueToString(elem)) cell.setCellValue(DataType.valueToString(elem).toDouble) - case BigDecimalType(_, _) => + case BigDecimalType => doubleCheck(DataType.valueToString(elem)) cell.setCellValue(DataType.valueToString(elem).toDouble) case _ => diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/utils/UJESClientUtils.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/utils/UJESClientUtils.scala index e31136f6ae..e75929ea8f 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/utils/UJESClientUtils.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/utils/UJESClientUtils.scala @@ -21,7 +21,6 @@ import org.apache.linkis.ujes.client.exception.UJESClientBuilderException import org.apache.linkis.ujes.client.request.JobExecuteAction.{EngineType, RunType} import org.apache.linkis.ujes.client.response.ResultSetResult -import java.math.{BigDecimal => JavaBigDecimal} import java.util import java.util.Locale @@ -30,7 +29,6 @@ import com.google.gson.{Gson, JsonObject} object UJESClientUtils { val gson: Gson = new Gson() - val DECIMAL_REGEX = "^decimal\\(\\s*\\d*\\s*,\\s*\\d*\\s*\\)".r.unanchored def toEngineType(engineType: String): EngineType = engineType match { case "spark" => EngineType.SPARK @@ -77,7 +75,7 @@ object UJESClientUtils { case "boolean" => value.toBoolean case "byte" => value.toByte case "bigint" => value.toLong - case "decimal" | DECIMAL_REGEX() => new JavaBigDecimal(value) + case "decimal" => value.toDouble case "array" => gson.fromJson(value, classOf[util.ArrayList[Object]]) case "map" => gson.fromJson(value, classOf[util.HashMap[Object, Object]]) case "struct" => gson.fromJson(value, classOf[JsonObject]) diff --git a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java index 8117c617de..0747a836c6 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java +++ b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java @@ -146,7 +146,7 @@ public static String getTypeStr(int type) { retVal = BinaryType.typeName(); break; case Types.DECIMAL: - retVal = new DecimalType("decimal", 3).typeName(); + retVal = DecimalType.typeName(); break; case Types.ARRAY: retVal = ArrayType.typeName(); diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala index dc4f1bfff0..1a0eecc794 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala @@ -77,7 +77,7 @@ object DolphinToSpark { case wds.BigIntType => LongType case wds.FloatType => FloatType case wds.DoubleType => DoubleType - case wds.DecimalType(_, _) => DecimalType(bigDecimalPrecision, bigDecimalScale) + case wds.DecimalType => DecimalType(bigDecimalPrecision, bigDecimalScale) case wds.DateType => DateType // case wds.TimestampType => TimestampType case wds.BinaryType => BinaryType