Skip to content

Commit

Permalink
1.6.7 (#145)
Browse files Browse the repository at this point in the history
* update base.json

* Split component JSON file

* rm @JsonIgnore

* Delete useless code

* Comment Log

* fix organ

* fix task

* update task model mpc-lr

* update project

* update task model JointStatisticalComponentTaskServiceImpl

* add log

* feat: mpc statistics component (#141)

* 1.6.6 develop (#125)

* add(sys):add collect api

* fix(fusion):update fusion api

* fix(csv):update csv api

* update db

* add fusion-simple

* update fusion-simple

* add resource field type grpc

* update resource

* update sqllite

* add log

* update sqllite

* update task mpc lr

* fix Collectors.toMap Duplicate key

* delete db project column "secretkey_id"

* Annotate trusted third-party code

* update grpc param

* update fusion

* fix 推理 -- 协作方资源

* add 推理 模板

* update infer

* fix

* update infer

* add 修改合作机构网关和公钥

* update data task resultPath

* update model task

* update change other organInfo

* update NN

* add health

* add health path

* update hfl-nn


* update sync dataset

* update organ resource

* update data_set address

* update change

* update organ change

* update model

* update xml

* update get model data

* update fusion

* update readme

* add yaml

* update base.json
  • Loading branch information
Zjj614 authored Jun 21, 2023
1 parent 9236050 commit fbfd415
Show file tree
Hide file tree
Showing 84 changed files with 1,906 additions and 1,967 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public void newDataset(NewDatasetRequest request, StreamObserver<NewDatasetRespo
responseObserver.onCompleted();
}


@Override
public void getDataset(GetDatasetRequest request, StreamObserver<GetDatasetResponse> responseObserver){
log.info("get ids:{}",request.getIdList().toString());
List<ByteString> ids = request.getIdList().asByteStringList();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
// @NacosPropertySource(dataId = "test", autoRefreshed = true),
// @NacosPropertySource(dataId = "test.yaml", autoRefreshed = true),
@NacosPropertySource(dataId = "base.json" ,autoRefreshed = true),
@NacosPropertySource(dataId = "components.json" ,autoRefreshed = true),
@NacosPropertySource(dataId = "database.yaml" ,autoRefreshed = true),
@NacosPropertySource(dataId = "redis.yaml" ,autoRefreshed = true)})
@SpringBootApplication(scanBasePackages="com.primihub")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,4 +68,4 @@ public BaseResultEntity display(String type,Integer operation){
return BaseResultEntity.success(marketService.display(type,operation));
}

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,30 @@ public class BaseConfiguration {
private String uploadUrlDirPrefix;
private String resultUrlDirPrefix;
private String runModelFileUrlDirPrefix;
private List<ModelComponent> modelComponents;
private String usefulToken;
private String taskEmailSubject;
// resource
/**
* resource
*/
private boolean displayDatabaseSourceType = false;
// auth
/**
* auth
*/
private Map<String, BaseAuthConfig> authConfigs;
// mail
/**
* mail
*/
private MailProperties mailProperties;
// Use in mail text content
/**
* Use in mail text content
*/
private String systemDomainName;
// loki
/**
* loki
*/
private LokiConfig lokiConfig;
/**
* Open the nacos template for debugging
*/
private Boolean openDynamicTuning = false;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package com.primihub.biz.config.base;

import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties;
import com.primihub.biz.entity.data.vo.ModelComponent;
import lombok.Getter;
import lombok.Setter;
import org.springframework.stereotype.Component;

import java.util.List;

@Getter
@Setter
@Component
@NacosConfigurationProperties(dataId = "components.json",autoRefreshed = true)
public class ComponentsConfiguration {
private List<ModelComponent> modelComponents;
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
Expand Down Expand Up @@ -70,7 +69,6 @@ public String generateUniqueCode(){
@PostConstruct
public void init(){
readNacosConfigOrganInfo();
// readNacosConfigCollectOrganInfo();
readResourceConfigCollectOrganInfo();
}

Expand Down Expand Up @@ -112,7 +110,7 @@ public void receiveConfigInfo(String config) {

public void readResourceConfigCollectOrganInfo(){
try {
org.springframework.core.io.Resource resource = new ClassPathResource("collectData.json");
org.springframework.core.io.Resource resource = new ClassPathResource(SysConstant.SYS_COLLECT_DATA_ORGAN_INFO_NAME);
InputStream is = resource.getInputStream();
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
Expand All @@ -130,39 +128,5 @@ public void readResourceConfigCollectOrganInfo(){
}catch (Exception e){
log.info("读取文件失败:{}",e.getMessage());
}

}

// public void readNacosConfigCollectOrganInfo(){
// try {
// String group=environment.getProperty("nacos.config.group");
// String serverAddr=environment.getProperty("nacos.config.server-addr");
// Properties properties = new Properties();
// properties.put("serverAddr",serverAddr);
// ConfigService configService= NacosFactory.createConfigService(properties);
// String collectDataContent=configService.getConfig(SysConstant.SYS_COLLECT_DATA_ORGAN_INFO_NAME,group,3000);
// log.info(" nacos collect_data data:{}",collectDataContent);
// if (StringUtils.isNotBlank(collectDataContent)){
// collectOrganList = JSON.parseArray(collectDataContent,SysCollectOrgan.class);
// }
// configService.addListener(SysConstant.SYS_COLLECT_DATA_ORGAN_INFO_NAME, group, new Listener() {
// @Override
// public Executor getExecutor() {
// return null;
// }
//
// @Override
// public void receiveConfigInfo(String config) {
// log.info(" nacos receiveConfigInfo organ_info data:{}",config);
// if (StringUtils.isNotBlank(config)){
// collectOrganList = JSON.parseArray(collectDataContent,SysCollectOrgan.class);
// }else {
// collectOrganList = null;
// }
// }
// });
// }catch (Exception e){
// log.info("nacos collect_data Exception:{}",e.getMessage());
// }
// }
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.primihub.biz.tool.TemplatesHelper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;

Expand All @@ -20,10 +21,16 @@
public class TemplatesConfig {
@Resource
private Environment environment;
@Autowired
private BaseConfiguration baseConfiguration;

@PostConstruct
public void readNacosFreemarkerPython(){
try {
if (!baseConfiguration.getOpenDynamicTuning()){
log.info("Close nacos template debugging");
return;
}
String group=environment.getProperty("nacos.config.group");
String serverAddr=environment.getProperty("nacos.config.server-addr");
Properties properties = new Properties();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public class AjCaptchaProperties {
/**
* 右下角水印文字(我的水印).
*/
private String waterMark = "我的水印";
private String waterMark = "Primihub";

/**
* 右下角水印字体(文泉驿正黑).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public class AjCaptchaServiceAutoConfiguration {
@Bean
@ConditionalOnMissingBean
public CaptchaService captchaService(@Qualifier("ajCaptchaProperties")AjCaptchaProperties prop) {
log.info("自定义配置项:{}", prop.toString());
// log.info("自定义配置项:{}", prop.toString());
Properties config = new Properties();
config.put(Const.CAPTCHA_CACHETYPE, prop.getCacheType().name());
config.put(Const.CAPTCHA_WATER_MARK, prop.getWaterMark());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ public class PrimaryDruidDataSourceWrapper extends DruidDataSource implements In
@NacosValue(value="${spring.datasource.druid.primary.filter.config.enabled}",autoRefreshed = true)
private Boolean filterConfigEnabled = false;

// private String passwordCallbackClassName;

public void setMaxWait(int maxWait) {
this.maxWait = maxWait;
Expand Down Expand Up @@ -54,11 +53,6 @@ public void setConnectionProperties(String connectionProperties) {
this.connectionProperties = connectionProperties;
}

// @Override
// public void setPasswordCallbackClassName(String passwordCallbackClassName) {
// this.passwordCallbackClassName = passwordCallbackClassName;
// }

@Override
public void afterPropertiesSet() throws Exception {
super.setUrl(url);
Expand All @@ -82,7 +76,6 @@ public void afterPropertiesSet() throws Exception {
if (filterConfigEnabled){
super.addFilters("config");
}
// super.setPasswordCallbackClassName(passwordCallbackClassName);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ public class SecondaryDruidDataSourceWrapper extends DruidDataSource implements
@NacosValue(value="${spring.datasource.druid.secondary.filter.config.enabled}",autoRefreshed = true)
private Boolean filterConfigEnabled = false;

// private String passwordCallbackClassName;

public void setMaxWait(int maxWait) {
this.maxWait = maxWait;
Expand Down Expand Up @@ -50,11 +49,6 @@ public void setConnectionProperties(String connectionProperties) {
this.connectionProperties = connectionProperties;
}

// @Override
// public void setPasswordCallbackClassName(String passwordCallbackClassName) {
// this.passwordCallbackClassName = passwordCallbackClassName;
// }

@Override
public void afterPropertiesSet() throws Exception {
super.setUrl(url);
Expand All @@ -78,7 +72,6 @@ public void afterPropertiesSet() throws Exception {
if (filterConfigEnabled){
super.addFilters("config");
}
// super.setPasswordCallbackClassName(passwordCallbackClassName);
}

}
Original file line number Diff line number Diff line change
@@ -1,34 +1,34 @@
package com.primihub.biz.config.grpc;

import com.primihub.biz.grpc.server.DataGrpcService;
import com.primihub.biz.grpc.server.TestGrpcService;
import com.primihub.biz.grpc.server.WorkGrpcService;
import com.primihub.biz.config.base.BaseConfiguration;
import io.grpc.Server;
import io.grpc.ServerBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.io.IOException;

@Slf4j
@Configuration
public class GrpcServerConfiguration {

@Bean(name="grpcServer")
public Server initGrpcServer(BaseConfiguration baseConfiguration){
Server server=ServerBuilder.forPort(baseConfiguration.getGrpcServerPort())
.addService(new TestGrpcService())
.addService(new WorkGrpcService())
.addService(new DataGrpcService())
.build();
try {
server.start();
} catch (IOException e) {
log.error("初始化GRPC端口失败",e);
}
return server;
}

}
//package com.primihub.biz.config.grpc;
//
//import com.primihub.biz.grpc.server.DataGrpcService;
//import com.primihub.biz.grpc.server.TestGrpcService;
//import com.primihub.biz.grpc.server.WorkGrpcService;
//import com.primihub.biz.config.base.BaseConfiguration;
//import io.grpc.Server;
//import io.grpc.ServerBuilder;
//import lombok.extern.slf4j.Slf4j;
//import org.springframework.context.annotation.Bean;
//import org.springframework.context.annotation.Configuration;
//
//import java.io.IOException;
//
//@Slf4j
//@Configuration
//public class GrpcServerConfiguration {
//
// @Bean(name="grpcServer")
// public Server initGrpcServer(BaseConfiguration baseConfiguration){
// Server server=ServerBuilder.forPort(baseConfiguration.getGrpcServerPort())
// .addService(new TestGrpcService())
// .addService(new WorkGrpcService())
// .addService(new DataGrpcService())
// .build();
// try {
// server.start();
// } catch (IOException e) {
// log.error("初始化GRPC端口失败",e);
// }
// return server;
// }
//
//}
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package com.primihub.biz.constant;

public class CommonConstant {
// sync Project api
public static final String PROJECT_SYNC_API_URL = "<address>/share/shareData/syncProject";
public static final String MODEL_SYNC_API_URL = "<address>/share/shareData/syncModel";
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,6 @@ public class DataConstant {
// ModelComponentService impl bean name suffix
public final static String COMPONENT_BEAN_NAME_SUFFIX = "ComponentTaskServiceImpl";

// Port number Range
public final static Long[] GUEST_PORT_RANGE = new Long[]{20000L,30000L};
public final static Long[] HOST_PORT_RANGE = new Long[]{40000L,50000L};

public final static String TASK_LOG_FILE_NAME = "taskLog.log";

}
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,4 @@ public class MarketConstant {
put("jobPosition_other","其他");
}};

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,6 @@ public class RedisKeyConstant {
public static final String SCHEDULE_FUSION_COPY_KEY="schedule:fusion_copy_<date>_<piece>";
public static final String VERIFICATION_CODE_TYPE_KEY="verification:code_<code_type>_<code_key>";

// Port number key
public final static String REQUEST_PORT_NUMBER = "model:port_number:<square>";

// task key
public final static String TASK_STATUS_KEY = "ts:<taskId>:<jobId>";
public final static String TASK_STATUS_LIST_KEY = "ts:list";
Expand Down
Loading

0 comments on commit fbfd415

Please sign in to comment.