Now the cdes are dynamicly loaded from Exareme2

HBPMedical · Oct 18, 2022 · 033ffb6 · 033ffb6
1 parent 954d538
commit 033ffb6
Show file tree

Hide file tree

Showing 9 changed files with 189 additions and 46 deletions.
diff --git a/config/application.tmpl b/config/application.tmpl
@@ -36,6 +36,8 @@ spring:
 services:
   mipengine:
     algorithmsUrl: {{ .Env.MIPENGINE_URL}}/algorithms
+    attributesUrl: {{ .Env.MIPENGINE_URL}}/data_models_attributes
+    cdesMetadataUrl: {{ .Env.MIPENGINE_URL}}/cdes_metadata
 
   exareme:
     queryExaremeUrl: {{ default .Env.EXAREME_URL "http://localhost:9090" }}/mining/query
@@ -83,4 +85,4 @@ endpoints:
   health:
     enabled: true
     endpoint: /health
-    sensitive: false
+    sensitive: false
diff --git a/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java b/src/main/java/eu/hbp/mip/controllers/AlgorithmsAPI.java
@@ -78,9 +78,12 @@ public ResponseEntity<List<ExaremeAlgorithmDTO>> getAlgorithms() {
 
         // Remove Exareme algorithms that exist in the Exareme2
         if (mipengineAlgorithms != null && exaremeAlgorithms != null){
+            int old_exareme_algorithm_size = exaremeAlgorithms.size();
+
             for (ExaremeAlgorithmDTO algorithm : mipengineAlgorithms) {
                 exaremeAlgorithms.removeIf(obj -> Objects.equals(obj.getName(), algorithm.getName()));
             }
+            logger.LogUserAction("Removed "+ (old_exareme_algorithm_size - exaremeAlgorithms.size()) +" deprecated exareme algorithms");
         }
 
         if (exaremeAlgorithms != null) {
@@ -109,15 +112,16 @@ public ResponseEntity<List<ExaremeAlgorithmDTO>> getAlgorithms() {
             logger.LogUserAction("The disabled algorithms could not be loaded.");
         }
 
-
-
         // Remove any disabled algorithm
         ArrayList<ExaremeAlgorithmDTO> allowedAlgorithms = new ArrayList<>();
         for (ExaremeAlgorithmDTO algorithm : algorithms) {
             if (!disabledAlgorithms.contains(algorithm.getName())) {
                 allowedAlgorithms.add(algorithm);
             }
         }
+
+        logger.LogUserAction("Removed "+ (algorithms.size() - allowedAlgorithms.size()) +" disabled algorithms");
+
         logger.LogUserAction("Successfully listed " + allowedAlgorithms.size() + " algorithms");
         return ResponseEntity.ok(allowedAlgorithms);
     }
@@ -138,15 +142,12 @@ public ArrayList<ExaremeAlgorithmDTO> getExaremeAlgorithms(Logger logger) {
                     new TypeToken<ArrayList<ExaremeAlgorithmDTO>>() {
                     }.getType()
             );
-        } catch (ConnectException e) {
-            logger.LogUserAction("An exception occurred: " + e.getMessage());
-            return null;
-        } catch (IOException e) {
+        } catch (Exception e) {
             logger.LogUserAction("An exception occurred: " + e.getMessage());
             return null;
         }
 
-        logger.LogUserAction("Completed, returned " + algorithms.size() + " algorithms.");
+        logger.LogUserAction("Completed, returned " + algorithms.size() + " Exareme algorithms.");
         return algorithms;
     }
 
@@ -166,18 +167,15 @@ public ArrayList<ExaremeAlgorithmDTO> getMIPEngineAlgorithms(Logger logger) {
                     new TypeToken<ArrayList<MIPEngineAlgorithmDTO>>() {
                     }.getType()
             );
-        } catch (ConnectException e) {
-            logger.LogUserAction("An exception occurred: " + e.getMessage());
-            return null;
-        } catch (IOException e) {
+        } catch (Exception e) {
             logger.LogUserAction("An exception occurred: " + e.getMessage());
             return null;
         }
 
         ArrayList<ExaremeAlgorithmDTO> algorithms = new ArrayList<>();
         mipEngineAlgorithms.forEach(mipEngineAlgorithm -> algorithms.add(new ExaremeAlgorithmDTO(mipEngineAlgorithm)));
 
-        logger.LogUserAction("Completed, returned " + algorithms.size() + " algorithms.");
+        logger.LogUserAction("Completed, returned  " + algorithms.size() + " Exareme2 algorithms.");
         return algorithms;
     }
 
@@ -256,4 +254,4 @@ List<String> getDisabledAlgorithms() throws IOException {
                 }.getType()
         );
     }
-}
+}
diff --git a/src/main/java/eu/hbp/mip/controllers/PathologiesAPI.java b/src/main/java/eu/hbp/mip/controllers/PathologiesAPI.java
@@ -2,24 +2,22 @@
 
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
+import eu.hbp.mip.models.DTOs.MIPEngineAttributesDTO;
+import eu.hbp.mip.models.DTOs.MetadataHierarchyDTO;
 import eu.hbp.mip.models.DTOs.PathologyDTO;
 import eu.hbp.mip.services.ActiveUserService;
-import eu.hbp.mip.utils.ClaimUtils;
-import eu.hbp.mip.utils.CustomResourceLoader;
-import eu.hbp.mip.utils.Exceptions.BadRequestException;
-import eu.hbp.mip.utils.InputStreamConverter;
-import eu.hbp.mip.utils.Logger;
+import eu.hbp.mip.utils.*;
 import io.swagger.annotations.Api;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.core.io.Resource;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.core.Authentication;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RestController;
 
 import java.io.IOException;
-import java.util.List;
+import java.net.ConnectException;
+import java.util.*;
 
 import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
 
@@ -35,41 +33,88 @@ public class PathologiesAPI {
     @Value("#{'${authentication.enabled}'}")
     private boolean authenticationIsEnabled;
 
-    @Value("#{'${files.pathologies_json}'}")
-    private String pathologiesFilePath;
+    @Value("#{'${services.mipengine.attributesUrl}'}")
+    private String mipengineAttributesUrl;
 
+    @Value("#{'${services.mipengine.cdesMetadataUrl}'}")
+    private String mipengineCDEsMetadataUrl;
     private final ActiveUserService activeUserService;
 
-    private final CustomResourceLoader resourceLoader;
-
-    public PathologiesAPI(ActiveUserService activeUserService, CustomResourceLoader resourceLoader) {
+    public PathologiesAPI(ActiveUserService activeUserService) {
         this.activeUserService = activeUserService;
-        this.resourceLoader = resourceLoader;
     }
 
     @RequestMapping(name = "/pathologies", method = RequestMethod.GET)
     public ResponseEntity<String> getPathologies(Authentication authentication) {
         Logger logger = new Logger(activeUserService.getActiveUser().getUsername(), "(GET) /pathologies");
         logger.LogUserAction("Loading pathologies ...");
 
-        // Load pathologies from file
-        Resource resource = resourceLoader.getResource(pathologiesFilePath);
-        List<PathologyDTO> allPathologies;
-        try {
-            allPathologies = gson.fromJson(InputStreamConverter.convertInputStreamToString(resource.getInputStream()), new TypeToken<List<PathologyDTO>>() {
-            }.getType());
-        } catch (IOException e) {
-            logger.LogUserAction("Unable to load pathologies");
-            throw new BadRequestException("The pathologies could not be loaded.");
+        Map<String, List<PathologyDTO.PathologyDatasetDTO>> datasetsPerPathology = getMIPEngineDatasetsPerPathology(logger);
+        System.out.println(datasetsPerPathology);
+
+        Map<String, MIPEngineAttributesDTO> mipEnginePathologyAttributes = getMIPEnginePathologyAttributes(logger);
+        System.out.println(mipEnginePathologyAttributes);
+
+        List<PathologyDTO> pathologyDTOS = new ArrayList<>();
+        for (String pathology : mipEnginePathologyAttributes.keySet()) {
+            pathologyDTOS.add(new PathologyDTO(pathology, mipEnginePathologyAttributes.get(pathology), datasetsPerPathology.get(pathology)));
         }
+        System.out.println(pathologyDTOS);
 
         // If authentication is disabled return everything
         if (!authenticationIsEnabled) {
-            logger.LogUserAction("Successfully loaded " + allPathologies.size() + " pathologies");
-            return ResponseEntity.ok().body(gson.toJson(allPathologies));
+            logger.LogUserAction("Successfully loaded " + pathologyDTOS.size() + " pathologies");
+            return ResponseEntity.ok().body(gson.toJson(pathologyDTOS));
         }
 
         logger.LogUserAction("Successfully loaded all authorized pathologies");
-        return ResponseEntity.ok().body(ClaimUtils.getAuthorizedPathologies(logger,  authentication, allPathologies));
+        return ResponseEntity.ok().body(gson.toJson(ClaimUtils.getAuthorizedPathologies(logger, authentication, pathologyDTOS)));
+    }
+
+    public Map<String, List<PathologyDTO.PathologyDatasetDTO>> getMIPEngineDatasetsPerPathology(Logger logger) {
+        Map<String, Map<String, MetadataHierarchyDTO.CommonDataElement>> mipEngineCDEsMetadata;
+        // Get MIPEngine algorithms
+        try {
+            StringBuilder response = new StringBuilder();
+            HTTPUtil.sendGet(mipengineCDEsMetadataUrl, response);
+            mipEngineCDEsMetadata = gson.fromJson(
+                    response.toString(),
+                    new TypeToken<HashMap<String, Map<String, MetadataHierarchyDTO.CommonDataElement>>>() {
+                    }.getType()
+            );
+        } catch (Exception e) {
+            logger.LogUserAction("An exception occurred: " + e.getMessage());
+            return null;
+        }
+
+        Map<String, List<PathologyDTO.PathologyDatasetDTO>> datasetsPerPathology = new HashMap<>();
+
+        mipEngineCDEsMetadata.forEach( (pathology, cdePerDataset) ->  {
+            List<PathologyDTO.PathologyDatasetDTO> pathologyDatasetDTOS = new ArrayList<>();
+            cdePerDataset.forEach((dataset, cde) ->  pathologyDatasetDTOS.add(new PathologyDTO.PathologyDatasetDTO(dataset, cde.getLabel())));
+            datasetsPerPathology.put(pathology, pathologyDatasetDTOS);
+        });
+
+
+        return datasetsPerPathology;
+    }
+
+    public Map<String, MIPEngineAttributesDTO> getMIPEnginePathologyAttributes(Logger logger) {
+        Map<String, MIPEngineAttributesDTO> mipEnginePathologyAttributes;
+        // Get MIPEngine algorithms
+        try {
+            StringBuilder response = new StringBuilder();
+            HTTPUtil.sendGet(mipengineAttributesUrl, response);
+            mipEnginePathologyAttributes = gson.fromJson(
+                    response.toString(),
+                    new TypeToken<HashMap<String, MIPEngineAttributesDTO>>() {
+                    }.getType()
+            );
+        } catch (Exception e) {
+            logger.LogUserAction("An exception occurred: " + e.getMessage());
+            return null;
+        }
+
+        return mipEnginePathologyAttributes;
     }
 }
diff --git a/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAttributesDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/MIPEngineAttributesDTO.java
@@ -0,0 +1,20 @@
+package eu.hbp.mip.models.DTOs;
+
+import com.google.gson.annotations.SerializedName;
+import eu.hbp.mip.models.DTOs.MetadataHierarchyDTO;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.List;
+import java.util.Map;
+
+@Data
+@AllArgsConstructor
+public class MIPEngineAttributesDTO {
+    @SerializedName("properties")
+    private Map<String, List<MetadataHierarchyDTO>> properties;
+
+    @SerializedName("tags")
+    private Object tags;
+
+}
diff --git a/src/main/java/eu/hbp/mip/models/DTOs/MetadataHierarchyDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/MetadataHierarchyDTO.java
@@ -0,0 +1,57 @@
+package eu.hbp.mip.models.DTOs;
+
+import com.google.gson.annotations.SerializedName;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.Hashtable;
+import java.util.List;
+import java.util.Optional;
+
+@Data
+@AllArgsConstructor
+public class MetadataHierarchyDTO {
+
+    @SerializedName("variables")
+    private List<CommonDataElement> variables;
+
+    @SerializedName("code")
+    private String code;
+
+    @SerializedName("groups")
+    private Object groups;
+
+    @SerializedName("label")
+    private String label;
+    @Data
+    @AllArgsConstructor
+    public static class CommonDataElement {
+        @SerializedName("isCategorical")
+        private Boolean isCategorical;
+
+        @SerializedName("code")
+        private String code;
+
+        @SerializedName("sql_type")
+        private String sql_type;
+
+        @SerializedName("description")
+        private String description;
+
+        @SerializedName("enumerations")
+        private Object enumerations;
+
+        @SerializedName("label")
+        private String label;
+
+        @SerializedName("units")
+        private String units;
+
+        @SerializedName("type")
+        private String type;
+
+        @SerializedName("methodology")
+        private String methodology;
+
+    }
+}
diff --git a/src/main/java/eu/hbp/mip/models/DTOs/PathologyDTO.java b/src/main/java/eu/hbp/mip/models/DTOs/PathologyDTO.java
@@ -4,7 +4,9 @@
 import lombok.AllArgsConstructor;
 import lombok.Data;
 
+import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 
 @Data
 @AllArgsConstructor
@@ -21,14 +23,31 @@ public class PathologyDTO {
     private String label;
 
     @SerializedName("metadataHierarchy")
-    private Object metadataHierarchy;
+    private MetadataHierarchyDTO metadataHierarchyDTO;
 
     @SerializedName("datasets")
     private List<PathologyDatasetDTO> datasets;
 
     public PathologyDTO(){
 
     }
+
+
+    public PathologyDTO(String pathology, MIPEngineAttributesDTO mipEngineAttributesDTO, List<PathologyDatasetDTO> pathologyDatasetDTOS) {
+        MetadataHierarchyDTO metadataHierarchyDTO = mipEngineAttributesDTO.getProperties().get("cdes").get(0);
+        List<MetadataHierarchyDTO.CommonDataElement> variables = metadataHierarchyDTO.getVariables();
+        variables.stream().filter(cde -> cde.getCode().equals("dataset")).
+                findAny().ifPresent(cde -> cde.setEnumerations(pathologyDatasetDTOS));
+        metadataHierarchyDTO.setVariables(variables);
+
+        List<String> pathology_info = Arrays.asList(pathology.split(":", 2));
+        this.code = pathology_info.get(0);
+        this.version = pathology_info.get(1);
+        this.metadataHierarchyDTO = metadataHierarchyDTO;
+        this.label = metadataHierarchyDTO.getLabel();
+        this.datasets = pathologyDatasetDTOS;
+    }
+
     @Data
     @AllArgsConstructor
     public static class PathologyDatasetDTO {

diff --git a/src/main/java/eu/hbp/mip/services/ExperimentService.java b/src/main/java/eu/hbp/mip/services/ExperimentService.java
@@ -124,7 +124,7 @@ public ExperimentDTO getExperiment(Authentication authentication, String uuid, L
         logger.LogUserAction("Loading Experiment with uuid : " + uuid);
 
         experimentDAO = experimentRepository.loadExperiment(uuid, logger);
-        if (               
+        if (
             authenticationIsEnabled
             && !experimentDAO.isShared()
             && !experimentDAO.getCreatedBy().getUsername().equals(user.getUsername())

diff --git a/src/main/java/eu/hbp/mip/utils/ClaimUtils.java b/src/main/java/eu/hbp/mip/utils/ClaimUtils.java
@@ -54,16 +54,16 @@ public static boolean validateAccessRightsOnExperiments(Authentication authentic
         return  hasRoleAccess(authorities, ClaimUtils.allExperimentsAllowedClaim(), logger);
     }
 
-    public static String getAuthorizedPathologies(Logger logger, Authentication authentication,
-                                                  List<PathologyDTO> allPathologies) {
+    public static List<PathologyDTO> getAuthorizedPathologies(Logger logger, Authentication authentication,
+                                                              List<PathologyDTO> allPathologies) {
         // --- Providing only the allowed pathologies/datasets to the user  ---
         logger.LogUserAction("Filter out the unauthorised datasets.");
 
         ArrayList<String> authorities = getKeycloakAuthorities(authentication, logger);
 
         // If the "dataset_all" claim exists then return everything
         if (hasRoleAccess(authorities, ClaimUtils.allDatasetsAllowedClaim(), logger)) {
-            return gson.toJson(allPathologies);
+            return allPathologies;
         }
 
         List<PathologyDTO> userPathologies = new ArrayList<>();
@@ -83,13 +83,13 @@ public static String getAuthorizedPathologies(Logger logger, Authentication auth
                 PathologyDTO userPathology = new PathologyDTO();
                 userPathology.setCode(curPathology.getCode());
                 userPathology.setLabel(curPathology.getLabel());
-                userPathology.setMetadataHierarchy(curPathology.getMetadataHierarchy());
+                userPathology.setMetadataHierarchyDTO(curPathology.getMetadataHierarchyDTO());
                 userPathology.setDatasets(userPathologyDatasets);
                 userPathologies.add(userPathology);
             }
         }
 
-        return gson.toJson(userPathologies);
+        return userPathologies;
     }
 
     private static boolean  hasRoleAccess(ArrayList<String> authorities, String role, Logger logger)