From 033ffb699c06d28ab311aeaaa9c4f6c969904bdf Mon Sep 17 00:00:00 2001 From: kfilippopolitis Date: Tue, 18 Oct 2022 14:43:08 +0300 Subject: [PATCH] Now the cdes are dynamicly loaded from Exareme2 --- config/application.tmpl | 4 +- .../eu/hbp/mip/controllers/AlgorithmsAPI.java | 24 +++-- .../hbp/mip/controllers/PathologiesAPI.java | 95 ++++++++++++++----- .../models/DTOs/MIPEngineAttributesDTO.java | 20 ++++ .../mip/models/DTOs/MetadataHierarchyDTO.java | 57 +++++++++++ .../eu/hbp/mip/models/DTOs/PathologyDTO.java | 21 +++- .../hbp/mip/services/ExperimentService.java | 2 +- .../java/eu/hbp/mip/utils/ClaimUtils.java | 10 +- src/main/resources/application.yml | 2 + 9 files changed, 189 insertions(+), 46 deletions(-) create mode 100644 src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAttributesDTO.java create mode 100644 src/main/java/eu/hbp/mip/models/DTOs/MetadataHierarchyDTO.java diff --git a/config/application.tmpl b/config/application.tmpl index e6af51b24..e42153df2 100644 --- a/config/application.tmpl +++ b/config/application.tmpl @@ -36,6 +36,8 @@ spring: services: mipengine: algorithmsUrl: {{ .Env.MIPENGINE_URL}}/algorithms + attributesUrl: {{ .Env.MIPENGINE_URL}}/data_models_attributes + cdesMetadataUrl: {{ .Env.MIPENGINE_URL}}/cdes_metadata exareme: queryExaremeUrl: {{ default .Env.EXAREME_URL "http://localhost:9090" }}/mining/query @@ -83,4 +85,4 @@ endpoints: health: enabled: true endpoint: /health - sensitive: false \ No newline at end of file + sensitive: false diff --git a/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java b/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java index 1fb02b09d..44e40651d 100644 --- a/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java +++ b/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java @@ -78,9 +78,12 @@ public ResponseEntity> getAlgorithms() { // Remove Exareme algorithms that exist in the Exareme2 if (mipengineAlgorithms != null && exaremeAlgorithms != null){ + int old_exareme_algorithm_size = exaremeAlgorithms.size(); + for (ExaremeAlgorithmDTO algorithm : mipengineAlgorithms) { exaremeAlgorithms.removeIf(obj -> Objects.equals(obj.getName(), algorithm.getName())); } + logger.LogUserAction("Removed "+ (old_exareme_algorithm_size - exaremeAlgorithms.size()) +" deprecated exareme algorithms"); } if (exaremeAlgorithms != null) { @@ -109,8 +112,6 @@ public ResponseEntity> getAlgorithms() { logger.LogUserAction("The disabled algorithms could not be loaded."); } - - // Remove any disabled algorithm ArrayList allowedAlgorithms = new ArrayList<>(); for (ExaremeAlgorithmDTO algorithm : algorithms) { @@ -118,6 +119,9 @@ public ResponseEntity> getAlgorithms() { allowedAlgorithms.add(algorithm); } } + + logger.LogUserAction("Removed "+ (algorithms.size() - allowedAlgorithms.size()) +" disabled algorithms"); + logger.LogUserAction("Successfully listed " + allowedAlgorithms.size() + " algorithms"); return ResponseEntity.ok(allowedAlgorithms); } @@ -138,15 +142,12 @@ public ArrayList getExaremeAlgorithms(Logger logger) { new TypeToken>() { }.getType() ); - } catch (ConnectException e) { - logger.LogUserAction("An exception occurred: " + e.getMessage()); - return null; - } catch (IOException e) { + } catch (Exception e) { logger.LogUserAction("An exception occurred: " + e.getMessage()); return null; } - logger.LogUserAction("Completed, returned " + algorithms.size() + " algorithms."); + logger.LogUserAction("Completed, returned " + algorithms.size() + " Exareme algorithms."); return algorithms; } @@ -166,10 +167,7 @@ public ArrayList getMIPEngineAlgorithms(Logger logger) { new TypeToken>() { }.getType() ); - } catch (ConnectException e) { - logger.LogUserAction("An exception occurred: " + e.getMessage()); - return null; - } catch (IOException e) { + } catch (Exception e) { logger.LogUserAction("An exception occurred: " + e.getMessage()); return null; } @@ -177,7 +175,7 @@ public ArrayList getMIPEngineAlgorithms(Logger logger) { ArrayList algorithms = new ArrayList<>(); mipEngineAlgorithms.forEach(mipEngineAlgorithm -> algorithms.add(new ExaremeAlgorithmDTO(mipEngineAlgorithm))); - logger.LogUserAction("Completed, returned " + algorithms.size() + " algorithms."); + logger.LogUserAction("Completed, returned " + algorithms.size() + " Exareme2 algorithms."); return algorithms; } @@ -256,4 +254,4 @@ List getDisabledAlgorithms() throws IOException { }.getType() ); } -} \ No newline at end of file +} diff --git a/src/main/java/eu/hbp/mip/controllers/PathologiesAPI.java b/src/main/java/eu/hbp/mip/controllers/PathologiesAPI.java index 5e9a9d344..e53d8d354 100644 --- a/src/main/java/eu/hbp/mip/controllers/PathologiesAPI.java +++ b/src/main/java/eu/hbp/mip/controllers/PathologiesAPI.java @@ -2,16 +2,13 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import eu.hbp.mip.models.DTOs.MIPEngineAttributesDTO; +import eu.hbp.mip.models.DTOs.MetadataHierarchyDTO; import eu.hbp.mip.models.DTOs.PathologyDTO; import eu.hbp.mip.services.ActiveUserService; -import eu.hbp.mip.utils.ClaimUtils; -import eu.hbp.mip.utils.CustomResourceLoader; -import eu.hbp.mip.utils.Exceptions.BadRequestException; -import eu.hbp.mip.utils.InputStreamConverter; -import eu.hbp.mip.utils.Logger; +import eu.hbp.mip.utils.*; import io.swagger.annotations.Api; import org.springframework.beans.factory.annotation.Value; -import org.springframework.core.io.Resource; import org.springframework.http.ResponseEntity; import org.springframework.security.core.Authentication; import org.springframework.web.bind.annotation.RequestMapping; @@ -19,7 +16,8 @@ import org.springframework.web.bind.annotation.RestController; import java.io.IOException; -import java.util.List; +import java.net.ConnectException; +import java.util.*; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; @@ -35,16 +33,15 @@ public class PathologiesAPI { @Value("#{'${authentication.enabled}'}") private boolean authenticationIsEnabled; - @Value("#{'${files.pathologies_json}'}") - private String pathologiesFilePath; + @Value("#{'${services.mipengine.attributesUrl}'}") + private String mipengineAttributesUrl; + @Value("#{'${services.mipengine.cdesMetadataUrl}'}") + private String mipengineCDEsMetadataUrl; private final ActiveUserService activeUserService; - private final CustomResourceLoader resourceLoader; - - public PathologiesAPI(ActiveUserService activeUserService, CustomResourceLoader resourceLoader) { + public PathologiesAPI(ActiveUserService activeUserService) { this.activeUserService = activeUserService; - this.resourceLoader = resourceLoader; } @RequestMapping(name = "/pathologies", method = RequestMethod.GET) @@ -52,24 +49,72 @@ public ResponseEntity getPathologies(Authentication authentication) { Logger logger = new Logger(activeUserService.getActiveUser().getUsername(), "(GET) /pathologies"); logger.LogUserAction("Loading pathologies ..."); - // Load pathologies from file - Resource resource = resourceLoader.getResource(pathologiesFilePath); - List allPathologies; - try { - allPathologies = gson.fromJson(InputStreamConverter.convertInputStreamToString(resource.getInputStream()), new TypeToken>() { - }.getType()); - } catch (IOException e) { - logger.LogUserAction("Unable to load pathologies"); - throw new BadRequestException("The pathologies could not be loaded."); + Map> datasetsPerPathology = getMIPEngineDatasetsPerPathology(logger); + System.out.println(datasetsPerPathology); + + Map mipEnginePathologyAttributes = getMIPEnginePathologyAttributes(logger); + System.out.println(mipEnginePathologyAttributes); + + List pathologyDTOS = new ArrayList<>(); + for (String pathology : mipEnginePathologyAttributes.keySet()) { + pathologyDTOS.add(new PathologyDTO(pathology, mipEnginePathologyAttributes.get(pathology), datasetsPerPathology.get(pathology))); } + System.out.println(pathologyDTOS); // If authentication is disabled return everything if (!authenticationIsEnabled) { - logger.LogUserAction("Successfully loaded " + allPathologies.size() + " pathologies"); - return ResponseEntity.ok().body(gson.toJson(allPathologies)); + logger.LogUserAction("Successfully loaded " + pathologyDTOS.size() + " pathologies"); + return ResponseEntity.ok().body(gson.toJson(pathologyDTOS)); } logger.LogUserAction("Successfully loaded all authorized pathologies"); - return ResponseEntity.ok().body(ClaimUtils.getAuthorizedPathologies(logger, authentication, allPathologies)); + return ResponseEntity.ok().body(gson.toJson(ClaimUtils.getAuthorizedPathologies(logger, authentication, pathologyDTOS))); + } + + public Map> getMIPEngineDatasetsPerPathology(Logger logger) { + Map> mipEngineCDEsMetadata; + // Get MIPEngine algorithms + try { + StringBuilder response = new StringBuilder(); + HTTPUtil.sendGet(mipengineCDEsMetadataUrl, response); + mipEngineCDEsMetadata = gson.fromJson( + response.toString(), + new TypeToken>>() { + }.getType() + ); + } catch (Exception e) { + logger.LogUserAction("An exception occurred: " + e.getMessage()); + return null; + } + + Map> datasetsPerPathology = new HashMap<>(); + + mipEngineCDEsMetadata.forEach( (pathology, cdePerDataset) -> { + List pathologyDatasetDTOS = new ArrayList<>(); + cdePerDataset.forEach((dataset, cde) -> pathologyDatasetDTOS.add(new PathologyDTO.PathologyDatasetDTO(dataset, cde.getLabel()))); + datasetsPerPathology.put(pathology, pathologyDatasetDTOS); + }); + + + return datasetsPerPathology; + } + + public Map getMIPEnginePathologyAttributes(Logger logger) { + Map mipEnginePathologyAttributes; + // Get MIPEngine algorithms + try { + StringBuilder response = new StringBuilder(); + HTTPUtil.sendGet(mipengineAttributesUrl, response); + mipEnginePathologyAttributes = gson.fromJson( + response.toString(), + new TypeToken>() { + }.getType() + ); + } catch (Exception e) { + logger.LogUserAction("An exception occurred: " + e.getMessage()); + return null; + } + + return mipEnginePathologyAttributes; } } diff --git a/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAttributesDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAttributesDTO.java new file mode 100644 index 000000000..2f81aee6a --- /dev/null +++ b/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAttributesDTO.java @@ -0,0 +1,20 @@ +package eu.hbp.mip.models.DTOs; + +import com.google.gson.annotations.SerializedName; +import eu.hbp.mip.models.DTOs.MetadataHierarchyDTO; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +@AllArgsConstructor +public class MIPEngineAttributesDTO { + @SerializedName("properties") + private Map> properties; + + @SerializedName("tags") + private Object tags; + +} diff --git a/src/main/java/eu/hbp/mip/models/DTOs/MetadataHierarchyDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/MetadataHierarchyDTO.java new file mode 100644 index 000000000..e0db0c70d --- /dev/null +++ b/src/main/java/eu/hbp/mip/models/DTOs/MetadataHierarchyDTO.java @@ -0,0 +1,57 @@ +package eu.hbp.mip.models.DTOs; + +import com.google.gson.annotations.SerializedName; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.Hashtable; +import java.util.List; +import java.util.Optional; + +@Data +@AllArgsConstructor +public class MetadataHierarchyDTO { + + @SerializedName("variables") + private List variables; + + @SerializedName("code") + private String code; + + @SerializedName("groups") + private Object groups; + + @SerializedName("label") + private String label; + @Data + @AllArgsConstructor + public static class CommonDataElement { + @SerializedName("isCategorical") + private Boolean isCategorical; + + @SerializedName("code") + private String code; + + @SerializedName("sql_type") + private String sql_type; + + @SerializedName("description") + private String description; + + @SerializedName("enumerations") + private Object enumerations; + + @SerializedName("label") + private String label; + + @SerializedName("units") + private String units; + + @SerializedName("type") + private String type; + + @SerializedName("methodology") + private String methodology; + + } +} diff --git a/src/main/java/eu/hbp/mip/models/DTOs/PathologyDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/PathologyDTO.java index c1b0f434f..8771aefd1 100644 --- a/src/main/java/eu/hbp/mip/models/DTOs/PathologyDTO.java +++ b/src/main/java/eu/hbp/mip/models/DTOs/PathologyDTO.java @@ -4,7 +4,9 @@ import lombok.AllArgsConstructor; import lombok.Data; +import java.util.Arrays; import java.util.List; +import java.util.Optional; @Data @AllArgsConstructor @@ -21,7 +23,7 @@ public class PathologyDTO { private String label; @SerializedName("metadataHierarchy") - private Object metadataHierarchy; + private MetadataHierarchyDTO metadataHierarchyDTO; @SerializedName("datasets") private List datasets; @@ -29,6 +31,23 @@ public class PathologyDTO { public PathologyDTO(){ } + + + public PathologyDTO(String pathology, MIPEngineAttributesDTO mipEngineAttributesDTO, List pathologyDatasetDTOS) { + MetadataHierarchyDTO metadataHierarchyDTO = mipEngineAttributesDTO.getProperties().get("cdes").get(0); + List variables = metadataHierarchyDTO.getVariables(); + variables.stream().filter(cde -> cde.getCode().equals("dataset")). + findAny().ifPresent(cde -> cde.setEnumerations(pathologyDatasetDTOS)); + metadataHierarchyDTO.setVariables(variables); + + List pathology_info = Arrays.asList(pathology.split(":", 2)); + this.code = pathology_info.get(0); + this.version = pathology_info.get(1); + this.metadataHierarchyDTO = metadataHierarchyDTO; + this.label = metadataHierarchyDTO.getLabel(); + this.datasets = pathologyDatasetDTOS; + } + @Data @AllArgsConstructor public static class PathologyDatasetDTO { diff --git a/src/main/java/eu/hbp/mip/services/ExperimentService.java b/src/main/java/eu/hbp/mip/services/ExperimentService.java index 12d923f12..de27f46fe 100644 --- a/src/main/java/eu/hbp/mip/services/ExperimentService.java +++ b/src/main/java/eu/hbp/mip/services/ExperimentService.java @@ -124,7 +124,7 @@ public ExperimentDTO getExperiment(Authentication authentication, String uuid, L logger.LogUserAction("Loading Experiment with uuid : " + uuid); experimentDAO = experimentRepository.loadExperiment(uuid, logger); - if ( + if ( authenticationIsEnabled && !experimentDAO.isShared() && !experimentDAO.getCreatedBy().getUsername().equals(user.getUsername()) diff --git a/src/main/java/eu/hbp/mip/utils/ClaimUtils.java b/src/main/java/eu/hbp/mip/utils/ClaimUtils.java index f5676b52c..df2a75abf 100644 --- a/src/main/java/eu/hbp/mip/utils/ClaimUtils.java +++ b/src/main/java/eu/hbp/mip/utils/ClaimUtils.java @@ -54,8 +54,8 @@ public static boolean validateAccessRightsOnExperiments(Authentication authentic return hasRoleAccess(authorities, ClaimUtils.allExperimentsAllowedClaim(), logger); } - public static String getAuthorizedPathologies(Logger logger, Authentication authentication, - List allPathologies) { + public static List getAuthorizedPathologies(Logger logger, Authentication authentication, + List allPathologies) { // --- Providing only the allowed pathologies/datasets to the user --- logger.LogUserAction("Filter out the unauthorised datasets."); @@ -63,7 +63,7 @@ public static String getAuthorizedPathologies(Logger logger, Authentication auth // If the "dataset_all" claim exists then return everything if (hasRoleAccess(authorities, ClaimUtils.allDatasetsAllowedClaim(), logger)) { - return gson.toJson(allPathologies); + return allPathologies; } List userPathologies = new ArrayList<>(); @@ -83,13 +83,13 @@ public static String getAuthorizedPathologies(Logger logger, Authentication auth PathologyDTO userPathology = new PathologyDTO(); userPathology.setCode(curPathology.getCode()); userPathology.setLabel(curPathology.getLabel()); - userPathology.setMetadataHierarchy(curPathology.getMetadataHierarchy()); + userPathology.setMetadataHierarchyDTO(curPathology.getMetadataHierarchyDTO()); userPathology.setDatasets(userPathologyDatasets); userPathologies.add(userPathology); } } - return gson.toJson(userPathologies); + return userPathologies; } private static boolean hasRoleAccess(ArrayList authorities, String role, Logger logger) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 32ed7ce84..e301494d6 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -36,6 +36,8 @@ spring: services: mipengine: algorithmsUrl: "http://127.0.0.1:5000/algorithms" + attributesUrl: "http://127.0.0.1:5000/data_models_attributes" + cdesMetadataUrl: "http://127.0.0.1:5000/cdes_metadata" exareme: queryExaremeUrl: "http://127.0.0.1:9090/mining/query" algorithmsUrl: "http://127.0.0.1:9090/mining/algorithms.json"