Merge pull request #85 from likehabits/develop

update run model ftl
primihub · Jan 18, 2023 · 6057b2c · 6057b2c
2 parents d8d8acc + d2d26ed
commit 6057b2c
Show file tree

Hide file tree

Showing 18 changed files with 1,695 additions and 1,316 deletions.
diff --git a/...ce/application/src/main/java/com/primihub/application/controller/test/TestController.java b/...ce/application/src/main/java/com/primihub/application/controller/test/TestController.java
@@ -4,21 +4,24 @@
 import com.primihub.biz.config.mq.SingleTaskChannel;
 import com.primihub.biz.config.test.TestConfiguration;
 import com.primihub.biz.config.test.TestYamlConfiguration;
-import com.primihub.biz.entity.base.BaseFunctionHandleEntity;
-import com.primihub.biz.entity.base.BaseFunctionHandleEnum;
-import com.primihub.biz.entity.base.BaseJsonParam;
-import com.primihub.biz.entity.base.BaseResultEntity;
+import com.primihub.biz.entity.base.*;
+import com.primihub.biz.entity.data.base.ResourceFileData;
 import com.primihub.biz.repository.primaryredis.sys.SysAuthPrimaryRedisRepository;
+import com.primihub.biz.service.data.DataResourceService;
 import com.primihub.biz.service.test.TestService;
+import com.primihub.biz.util.FileUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.messaging.support.MessageBuilder;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
 import javax.servlet.http.HttpServletRequest;
+import java.io.File;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 @Slf4j
@@ -35,6 +38,8 @@ public class TestController {
     private SysAuthPrimaryRedisRepository sysAuthPrimaryRedisRepository;
     @Autowired
     private SingleTaskChannel singleTaskChannel;
+    @Autowired
+    private DataResourceService dataResourceService;
 
     @RequestMapping("/testPublish")
     public BaseResultEntity testPublish(){
@@ -103,6 +108,21 @@ public BaseResultEntity formatResources(String tag){
         return BaseResultEntity.success();
     }
 
+    @RequestMapping("/testFile")
+    public BaseResultEntity testFile(String filePath,Integer severalLines) throws Exception {
+        if (StringUtils.isBlank(filePath))
+            return BaseResultEntity.failure(BaseResultEnum.DATA_EDIT_FAIL);
+        ResourceFileData resourceFileData = dataResourceService.getResourceFileData(filePath);
+        return BaseResultEntity.success(resourceFileData);
+    }
+
+    @RequestMapping("/testFileMd5")
+    public BaseResultEntity testFileMd5(String filePath){
+        if (StringUtils.isBlank(filePath))
+            return BaseResultEntity.failure(BaseResultEnum.DATA_EDIT_FAIL);
+        return BaseResultEntity.success(FileUtil.md5HashCode(new File(filePath)));
+    }
+
 
 
 

diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/constant/DataConstant.java b/primihub-service/biz/src/main/java/com/primihub/biz/constant/DataConstant.java
@@ -50,7 +50,7 @@ public class DataConstant {
     public final static Long[] HOST_PORT_RANGE = new Long[]{40000L,50000L};
 
     public final static Long GRPC_SERVER_TIMEOUT = 24L * 60L * 60L * 1000L;
-    public final static Long GRPC_FILE_TIMEOUT = 5L * 60L * 1000L;
+    public final static Long GRPC_FILE_TIMEOUT = 30L * 1000L;
 
     public final static String TASK_LOG_FILE_NAME = "taskLog.log";
 

diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/entity/data/base/ResourceFileData.java b/primihub-service/biz/src/main/java/com/primihub/biz/entity/data/base/ResourceFileData.java
@@ -0,0 +1,42 @@
+package com.primihub.biz.entity.data.base;
+
+import lombok.Data;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Data
+public class ResourceFileData {
+
+    private String field;
+
+    private List<String> fieldList;
+
+    private Integer fieldSize = 0;
+
+    private Integer fileContentSize = 0;
+
+    private Integer resourceFileYRow = 0;
+
+    private boolean fileContainsY = false;
+
+    public Integer yIndex = 0;
+
+    public void setField(String field) {
+        this.field = field;
+        fieldList = Arrays.stream(field.split(",")).collect(Collectors.toList());
+        fieldSize = fieldList.size();
+        if (fieldList.contains("y")){
+            fileContainsY = true;
+            yIndex = fieldList.indexOf("y");
+        }
+    }
+
+    public void addContentSzie(){
+        fileContentSize++;
+    }
+    public void addFileYRowSzie(){
+        resourceFileYRow++;
+    }
+}
diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataAsyncService.java b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataAsyncService.java
@@ -108,6 +108,7 @@ public void setApplicationContext(ApplicationContext applicationContext) throws
     private DataTaskMonitorService dataTaskMonitorService;
 
 
+
     public BaseResultEntity executeBeanMethod(boolean isCheck,DataComponentReq req, ComponentTaskReq taskReq){
         String baenName = req.getComponentCode()+ DataConstant.COMPONENT_BEAN_NAME_SUFFIX;
         log.info("execute : {}",baenName);
@@ -449,10 +450,11 @@ public void runReasoning(DataReasoning dataReasoning,List<DataReasoningResource>
                 if (modelType.getVal().equals("2")){
                     map.put(DataConstant.PYTHON_GUEST_DATASET,guestDataset);
                     freemarkerContent = FreemarkerUtil.configurerCreateFreemarkerContent(DataConstant.FREEMARKER_PYTHON_HOMO_XGB_INFER_PATH, freeMarkerConfigurer, map);
+                    grpc(dataReasoning,dataTask,freemarkerContent,modelType.getVal(),2);
                 }else{
                     freemarkerContent = FreemarkerUtil.configurerCreateFreemarkerContent(DataConstant.FREEMARKER_PYTHON_HOMO_LR_INFER_PATH, freeMarkerConfigurer, map);
+                    grpc(dataReasoning,dataTask,freemarkerContent,modelType.getVal(),1);
                 }
-                grpc(dataReasoning,dataTask,freemarkerContent,modelType.getVal());
             }
         }
 
@@ -533,7 +535,7 @@ public Long[] getPortNumber(){
 
 
 
-    public void grpc(DataReasoning dataReasoning, DataTask dataTask, String freemarkerContent,String modelType){
+    public void grpc(DataReasoning dataReasoning, DataTask dataTask, String freemarkerContent,String modelType,int size){
         try {
             log.info(freemarkerContent);
             DataTask modelTask = dataTaskRepository.selectDataTaskByTaskId(dataReasoning.getTaskId());
@@ -569,7 +571,7 @@ public void grpc(DataReasoning dataReasoning, DataTask dataTask, String freemark
                     .setName("modelTask")
                     .setLanguage(Common.Language.PYTHON)
                     .setCode(ByteString.copyFrom(freemarkerContent.getBytes(StandardCharsets.UTF_8)))
-                    .setJobId(ByteString.copyFrom(dataTask.getTaskIdName().getBytes(StandardCharsets.UTF_8)))
+                    .setJobId(ByteString.copyFrom("1".getBytes(StandardCharsets.UTF_8)))
                     .setTaskId(ByteString.copyFrom(dataTask.getTaskIdName().getBytes(StandardCharsets.UTF_8)))
                     .build();
             log.info("grpc Common.Task :\n{}", task.toString());
@@ -581,13 +583,16 @@ public void grpc(DataReasoning dataReasoning, DataTask dataTask, String freemark
                     .build();
             PushTaskReply reply = workGrpcClient.run(o -> o.submitTask(request));
             log.info("grpc结果:{}", reply.toString());
-            if (reply.getRetCode()==0){
+//            if (reply.getRetCode()==0){
+                dataTaskMonitorService.verifyWhetherTheTaskIsSuccessfulAgain(dataTask, "1",size,dataTask.getTaskResultPath());
+            if (dataTask.getTaskState().equals(TaskStateEnum.SUCCESS.getStateType())){
                 dataReasoning.setReleaseDate(new Date());
-                dataTask.setTaskState(TaskStateEnum.SUCCESS.getStateType());
-            }else {
-                dataTask.setTaskState(TaskStateEnum.FAIL.getStateType());
-                dataTask.setTaskErrorMsg("运行失败:"+reply.getRetCode());
             }
+//                dataTask.setTaskState(TaskStateEnum.SUCCESS.getStateType());
+//            }else {
+//                dataTask.setTaskState(TaskStateEnum.FAIL.getStateType());
+//                dataTask.setTaskErrorMsg("运行失败:"+reply.getRetCode());
+//            }
         } catch (Exception e) {
             dataTask.setTaskState(TaskStateEnum.FAIL.getStateType());
             dataTask.setTaskErrorMsg(e.getMessage());

diff --git a/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataPsiService.java b/primihub-service/biz/src/main/java/com/primihub/biz/service/data/DataPsiService.java
@@ -65,7 +65,7 @@ public BaseResultEntity getPsiResourceAllocationList(PageReq req, String organId
             Set<Long> resourceIds = dataResources.stream().map(DataResource::getResourceId).collect(Collectors.toSet());
             List<DataFileField> dataFileField = dataResourceRepository.queryDataFileField(new HashMap() {{
                 put("resourceIds", resourceIds);
-                put("relevance", 1);
+//                put("relevance", 1);
             }});
             Map<Long, List<DataFileField>> fileFieldMap = dataFileField.stream().collect(Collectors.groupingBy(DataFileField::getResourceId));
             return BaseResultEntity.success(new PageDataEntity(count.intValue(),req.getPageSize(),req.getPageNo(),dataResources.stream().map(re-> DataResourceConvert.DataResourcePoConvertAllocationVo(re,fileFieldMap.get(re.getResourceId()),localOrganId)).collect(Collectors.toList())));