diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index 6ad90a6..18ddaed 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -1,6 +1,9 @@ package life.qbic; import ch.ethz.sis.openbis.generic.OpenBIS; +import java.util.HashMap; +import java.util.Map; +import life.qbic.io.PropertyReader; import life.qbic.io.commandline.CommandLineOptions; import life.qbic.model.Configuration; import life.qbic.model.download.AuthenticationException; @@ -18,6 +21,7 @@ public class App { private static final Logger LOG = LogManager.getLogger(App.class); + public static Map configProperties = new HashMap<>(); public static void main(String[] args) { LOG.debug("command line arguments: " + Arrays.deepToString(args)); @@ -26,6 +30,14 @@ public static void main(String[] args) { System.exit(exitCode); } + public static void readConfig() { + System.err.println("reading config"); + String configPath = CommandLineOptions.getConfigPath(); + if(configPath != null && !configPath.isEmpty()) { + configProperties = PropertyReader.getProperties(configPath); + } + } + /** * checks if the commandline parameter for reading out the password from the environment variable * is correctly provided @@ -57,7 +69,9 @@ public static OpenBIS loginToOpenBIS( char[] password, String user, String url, String dssUrl) { setupLog(); - OpenBIS authentication = new OpenBIS(url, dssUrl); + int generousTimeOut = 30*60*1000; //30 mins + + OpenBIS authentication = new OpenBIS(url, dssUrl, generousTimeOut); return tryLogin(authentication, user, password); } diff --git a/src/main/java/life/qbic/io/PropertyReader.java b/src/main/java/life/qbic/io/PropertyReader.java new file mode 100644 index 0000000..61e89a0 --- /dev/null +++ b/src/main/java/life/qbic/io/PropertyReader.java @@ -0,0 +1,44 @@ +package life.qbic.io; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.TreeMap; + +public class PropertyReader { + + public static TreeMap getProperties(String infile) { + + TreeMap properties = new TreeMap<>(); + BufferedReader bfr = null; + try { + bfr = new BufferedReader(new FileReader(new File(infile))); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + + String line; + while (true) { + try { + if ((line = bfr.readLine()) == null) + break; + } catch (IOException e) { + throw new RuntimeException(e); + } + if (!line.startsWith("#") && !line.isEmpty()) { + String[] property = line.trim().split("="); + properties.put(property[0].trim(), property[1].trim()); + } + } + + try { + bfr.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return(properties); + } +} diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java deleted file mode 100644 index eba85d1..0000000 --- a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java +++ /dev/null @@ -1,238 +0,0 @@ -package life.qbic.io.commandline; - -import static java.util.Objects.nonNull; -import static picocli.CommandLine.ArgGroup; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.StringJoiner; -import java.util.TreeMap; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import picocli.CommandLine; -import picocli.CommandLine.Option; - -public class AuthenticationOptions { - private static final Logger log = LogManager.getLogger(AuthenticationOptions.class); - - @Option( - names = {"-u", "--user"}, - description = "openBIS user name") - private String openbisUser; - @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. - OpenbisPasswordOptions openbisPasswordOptions; - @ArgGroup(multiplicity = "1") - SeekPasswordOptions seekPasswordOptions; - - @Option( - names = {"-as", "-as_url"}, - description = "OpenBIS ApplicationServer URL", - scope = CommandLine.ScopeType.INHERIT) - private String as_url; - - @Option( - names = {"-dss", "--dss_url"}, - description = "OpenBIS DatastoreServer URL", - scope = CommandLine.ScopeType.INHERIT) - private String dss_url; - - @Option( - names = {"-config", "--config_file"}, - description = "Config file path to provide server and user information.", - scope = CommandLine.ScopeType.INHERIT) - public String configPath; - @Option( - names = {"-su", "--seek-user"}, - description = "Seek user name (email)", - scope = CommandLine.ScopeType.INHERIT) - private String seekUser; - @Option( - names = {"-seek-server", "-seek_url"}, - description = "SEEK API URL", - scope = CommandLine.ScopeType.INHERIT) - private String seek_url; - - public String getOpenbisUser() { - if(openbisUser == null & configPath!=null && !configPath.isBlank()) { - openbisUser = ReadProperties.getProperties(configPath).get("user"); - } else { - log.error("No openBIS user provided."); - System.exit(2); - } - return openbisUser; - } - - public String getSeekUser() { - if(seekUser == null & configPath!=null && !configPath.isBlank()) { - seekUser = ReadProperties.getProperties(configPath).get("seek_user"); - } else { - log.error("No SEEK user/email provided."); - System.exit(2); - } - return seekUser; - } - - public String getSeekURL() { - if(seek_url == null & configPath!=null && !configPath.isBlank()) { - seek_url = ReadProperties.getProperties(configPath).get("seek_url"); - } else { - log.error("No URL to the SEEK address provided."); - System.exit(2); - } - return seek_url; - } - - public String getOpenbisDSS() { - if(dss_url == null & configPath!=null && !configPath.isBlank()) { - dss_url = ReadProperties.getProperties(configPath).get("dss"); - } - return dss_url; - } - - public String getOpenbisAS() { - if(as_url == null & configPath!=null && !configPath.isBlank()) { - as_url = ReadProperties.getProperties(configPath).get("as"); - } - return as_url; - } - - public char[] getSeekPassword() { - return seekPasswordOptions.getPassword(); - } - - public char[] getOpenbisPassword() { - return openbisPasswordOptions.getPassword(); - } - - public String getOpenbisBaseURL() throws MalformedURLException { - URL asURL = new URL(as_url); - String res = asURL.getProtocol()+ "://" +asURL.getHost(); - if(asURL.getPort()!=-1) { - res+=":"+asURL.getPort(); - } - System.err.println(res); - return res; - } - - /** - * official picocli documentation example - */ - static class OpenbisPasswordOptions { - @Option(names = "--openbis-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") - protected String passwordEnvironmentVariable = ""; - - @Option(names = "--openbis-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") - protected String passwordProperty = ""; - - @Option(names = "--openbis-pw", arity = "0", description = "please provide your openBIS password", interactive = true) - protected char[] password = null; - - /** - * Gets the password. If no password is provided, the program exits. - * @return the password provided by the user. - */ - char[] getPassword() { - if (nonNull(password)) { - return password; - } - // System.getProperty(String key) does not work for empty or blank keys. - if (!passwordProperty.isBlank()) { - String systemProperty = System.getProperty(passwordProperty); - if (nonNull(systemProperty)) { - return systemProperty.toCharArray(); - } - } - String environmentVariable = System.getenv(passwordEnvironmentVariable); - if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { - return environmentVariable.toCharArray(); - } - log.error("No password provided. Please provide your password."); - System.exit(2); - return null; // not reachable due to System.exit in previous line - } - } - - static class SeekPasswordOptions { - @Option(names = "--seek-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") - protected String passwordEnvironmentVariable = ""; - - @Option(names = "--seek-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") - protected String passwordProperty = ""; - - @Option(names = "--seek-pw", arity = "0", description = "please provide your SEEK password", interactive = true) - protected char[] password = null; - - /** - * Gets the password. If no password is provided, the program exits. - * @return the password provided by the user. - */ - char[] getPassword() { - if (nonNull(password)) { - return password; - } - // System.getProperty(String key) does not work for empty or blank keys. - if (!passwordProperty.isBlank()) { - String systemProperty = System.getProperty(passwordProperty); - if (nonNull(systemProperty)) { - return systemProperty.toCharArray(); - } - } - String environmentVariable = System.getenv(passwordEnvironmentVariable); - if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { - return environmentVariable.toCharArray(); - } - log.error("No password provided. Please provide your password."); - System.exit(2); - return null; // not reachable due to System.exit in previous line - } - - } - @Override - public String toString() { - return new StringJoiner(", ", AuthenticationOptions.class.getSimpleName() + "[", "]") - .add("user='" + openbisUser + "'") - .toString(); - //ATTENTION: do not expose the password here! - } - - public static class ReadProperties { - - public static TreeMap getProperties(String infile) { - - TreeMap properties = new TreeMap<>(); - BufferedReader bfr = null; - try { - bfr = new BufferedReader(new FileReader(new File(infile))); - } catch (FileNotFoundException e) { - throw new RuntimeException(e); - } - - String line; - while (true) { - try { - if ((line = bfr.readLine()) == null) - break; - } catch (IOException e) { - throw new RuntimeException(e); - } - if (!line.startsWith("#") && !line.isEmpty()) { - String[] property = line.trim().split("="); - properties.put(property[0].trim(), property[1].trim()); - } - } - - try { - bfr.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - - return(properties); - } - } -} \ No newline at end of file diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index b583d9c..332bd68 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -17,17 +17,24 @@ public class CommandLineOptions { private static final Logger LOG = LogManager.getLogger(CommandLineOptions.class); + @Option(names = {"-config", "--config_file"}, + description = "Config file path to provide server and user information.", + scope = CommandLine.ScopeType.INHERIT) + static String configPath; + @Option(names = {"-V", "--version"}, - versionHelp = true, - description = "print version information", - scope = CommandLine.ScopeType.INHERIT) + versionHelp = true, + description = "print version information", + scope = CommandLine.ScopeType.INHERIT) boolean versionRequested = false; - @Option( - names = {"-h", "--help"}, - usageHelp = true, - description = "display a help message and exit", - scope = CommandLine.ScopeType.INHERIT) + @Option(names = {"-h", "--help"}, + usageHelp = true, + description = "display a help message and exit", + scope = CommandLine.ScopeType.INHERIT) public boolean helpRequested = false; + public static String getConfigPath() { + return configPath; + } } diff --git a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java index 5be6b09..a80f861 100644 --- a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java +++ b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java @@ -6,6 +6,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import life.qbic.App; import life.qbic.io.PetabParser; import life.qbic.model.DatasetWithProperties; @@ -23,11 +24,12 @@ public class DownloadPetabCommand implements Runnable { @Parameters(arity = "1", paramLabel = "download path", description = "The local path where to store the downloaded data") private String outputPath; @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getOpenbisAS(), auth.getOpenbisDSS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); @@ -37,14 +39,16 @@ public void run() { return; } DatasetWithProperties result = new DatasetWithProperties(datasets.get(0)); - Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", + Set patientIDs = openbis.findPropertiesInSampleHierarchy("PATIENT_DKFZ_ID", result.getExperiment().getIdentifier()); - patientID.ifPresent(s -> result.addProperty("patientID", s)); + if(!patientIDs.isEmpty()) { + result.addProperty("patientIDs", String.join(",", patientIDs)); + } System.out.println("Found dataset, downloading."); System.out.println(); - openbis.downloadDataset(outputPath, datasetCode); + openbis.downloadDataset(outputPath, datasetCode, ""); PetabParser parser = new PetabParser(); try { diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index 9e384f9..bb2072f 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import life.qbic.App; import life.qbic.model.DatasetWithProperties; @@ -28,7 +29,7 @@ public class FindDatasetsCommand implements Runnable { @Option(arity = "1", paramLabel = "", description = "Optional openBIS spaces to filter samples", names = {"-s", "--space"}) private String space; @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); @Override public void run() { @@ -39,7 +40,8 @@ public void run() { } else { System.out.println("Querying experiment in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.listDatasetsOfExperiment(spaces, experimentCode).stream() .sorted(Comparator.comparing( @@ -47,9 +49,11 @@ public void run() { Collectors.toList()); Map properties = new HashMap<>(); if (!datasets.isEmpty()) { - Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", + Set patientIDs = openbis.findPropertiesInSampleHierarchy("PATIENT_DKFZ_ID", datasets.get(0).getExperiment().getIdentifier()); - patientID.ifPresent(s -> properties.put("Patient ID", s)); + if(!patientIDs.isEmpty()) { + properties.put("patientIDs", String.join(",", patientIDs)); + } } List datasetWithProperties = datasets.stream().map(dataSet -> { DatasetWithProperties ds = new DatasetWithProperties(dataSet); diff --git a/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java b/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java deleted file mode 100644 index 55923f7..0000000 --- a/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java +++ /dev/null @@ -1,30 +0,0 @@ -package life.qbic.io.commandline; - -import java.io.Console; -import java.util.Optional; - -public class OpenBISPasswordParser { - - - /** - * Retrieve the password from the system console - * - * @return the password read from the system console input - */ - public static String readPasswordFromConsole() { - Console console = System.console(); - char[] passwordChars = console.readPassword(); - return String.valueOf(passwordChars); - } - - /** - * @param variableName Name of given environment variable - * @return the password read from the environment variable - */ - public static Optional readPasswordFromEnvVariable(String variableName) { - - return Optional.ofNullable(System.getenv(variableName)); - - } - -} diff --git a/src/main/java/life/qbic/io/commandline/OpenbisAuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/OpenbisAuthenticationOptions.java new file mode 100644 index 0000000..9228d0b --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/OpenbisAuthenticationOptions.java @@ -0,0 +1,124 @@ +package life.qbic.io.commandline; + +import static java.util.Objects.nonNull; +import static picocli.CommandLine.ArgGroup; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.StringJoiner; +import life.qbic.App; +import life.qbic.io.PropertyReader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import picocli.CommandLine; +import picocli.CommandLine.Option; + +public class OpenbisAuthenticationOptions { + private static final Logger log = LogManager.getLogger(OpenbisAuthenticationOptions.class); + + @Option( + names = {"-u", "--user"}, + description = "openBIS user name") + private String openbisUser; + @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. + OpenbisPasswordOptions openbisPasswordOptions; + + @Option( + names = {"-as", "-as_url"}, + description = "OpenBIS ApplicationServer URL", + scope = CommandLine.ScopeType.INHERIT) + private String as_url; + + @Option( + names = {"-dss", "--dss_url"}, + description = "OpenBIS DatastoreServer URL", + scope = CommandLine.ScopeType.INHERIT) + private String dss_url; + + public String getOpenbisUser() { + if(openbisUser == null && App.configProperties.containsKey("user")) { + openbisUser = App.configProperties.get("user"); + } else { + log.error("No openBIS user provided."); + System.exit(2); + } + return openbisUser; + } + + public String getOpenbisDSS() { + if(dss_url == null && App.configProperties.containsKey("dss")) { + dss_url = App.configProperties.get("dss"); + } + return dss_url; + } + + public String getOpenbisAS() { + if(as_url == null && App.configProperties.containsKey("as")) { + as_url = App.configProperties.get("as"); + } + return as_url; + } + + public char[] getOpenbisPassword() { + return openbisPasswordOptions.getPassword(); + } + + public String getOpenbisBaseURL() throws MalformedURLException { + URL asURL = new URL(as_url); + String res = asURL.getProtocol()+ "://" +asURL.getHost(); + if(asURL.getPort()!=-1) { + res+=":"+asURL.getPort(); + } + return res; + } + + /** + * official picocli documentation example + */ + static class OpenbisPasswordOptions { + @Option(names = "--openbis-pw:env", arity = "1", paramLabel = "", + description = "provide the name of an environment variable to read in your password from") + protected String passwordEnvironmentVariable = ""; + + @Option(names = "--openbis-pw:prop", arity = "1", paramLabel = "", + description = "provide the name of a system property to read in your password from") + protected String passwordProperty = ""; + + @Option(names = "--openbis-pw", arity = "0", + description = "please provide your openBIS password", interactive = true) + protected char[] password = null; + + /** + * Gets the password. If no password is provided, the program exits. + * @return the password provided by the user. + */ + char[] getPassword() { + if (nonNull(password)) { + return password; + } + // System.getProperty(String key) does not work for empty or blank keys. + if (!passwordProperty.isBlank()) { + String systemProperty = System.getProperty(passwordProperty); + if (nonNull(systemProperty)) { + return systemProperty.toCharArray(); + } + } + String environmentVariable = System.getenv(passwordEnvironmentVariable); + if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { + return environmentVariable.toCharArray(); + } + log.error("No password provided. Please provide your password."); + System.exit(2); + return null; // not reachable due to System.exit in previous line + } + } + + @Override + public String toString() { + return new StringJoiner(", ", OpenbisAuthenticationOptions.class.getSimpleName() + "[", "]") + .add("user='" + openbisUser + "'") + .toString(); + //ATTENTION: do not expose the password here! + } + +} \ No newline at end of file diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index 2298693..228d615 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -31,7 +31,7 @@ public class SampleHierarchyCommand implements Runnable { @Option(arity = "1", paramLabel = "", description = "optional output path", names = {"-o", "--out"}) private String outpath; @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); @Override public void run() { @@ -43,7 +43,8 @@ public void run() { } else { summary.add("Querying samples in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); Map hierarchy = openbis.queryFullSampleHierarchy(spaces); diff --git a/src/main/java/life/qbic/io/commandline/SeekAuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/SeekAuthenticationOptions.java new file mode 100644 index 0000000..8a58d6e --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/SeekAuthenticationOptions.java @@ -0,0 +1,97 @@ +package life.qbic.io.commandline; + +import static java.util.Objects.nonNull; +import static picocli.CommandLine.ArgGroup; + +import java.util.StringJoiner; +import life.qbic.App; +import life.qbic.io.PropertyReader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import picocli.CommandLine; +import picocli.CommandLine.Option; + +public class SeekAuthenticationOptions { + private static final Logger log = LogManager.getLogger(SeekAuthenticationOptions.class); + + @ArgGroup(multiplicity = "1") + SeekPasswordOptions seekPasswordOptions; + + @Option( + names = {"-su", "--seek-user"}, + description = "Seek user name (email)", + scope = CommandLine.ScopeType.INHERIT) + private String seekUser; + @Option( + names = {"-seek-server", "-seek_url"}, + description = "SEEK API URL", + scope = CommandLine.ScopeType.INHERIT) + private String seek_url; + + public String getSeekUser() { + if(seekUser == null && App.configProperties.containsKey("seek_user")) { + seekUser = App.configProperties.get("seek_user"); + } else { + log.error("No SEEK user/email provided."); + System.exit(2); + } + return seekUser; + } + + public String getSeekURL() { + if(seek_url == null && App.configProperties.containsKey("seek_url")) { + seek_url = App.configProperties.get("seek_url"); + } else { + log.error("No URL to the SEEK address provided."); + System.exit(2); + } + return seek_url; + } + + public char[] getSeekPassword() { + return seekPasswordOptions.getPassword(); + } + + static class SeekPasswordOptions { + @Option(names = "--seek-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") + protected String passwordEnvironmentVariable = ""; + + @Option(names = "--seek-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") + protected String passwordProperty = ""; + + @Option(names = "--seek-pw", arity = "0", description = "please provide your SEEK password", interactive = true) + protected char[] password = null; + + /** + * Gets the password. If no password is provided, the program exits. + * @return the password provided by the user. + */ + char[] getPassword() { + if (nonNull(password)) { + return password; + } + // System.getProperty(String key) does not work for empty or blank keys. + if (!passwordProperty.isBlank()) { + String systemProperty = System.getProperty(passwordProperty); + if (nonNull(systemProperty)) { + return systemProperty.toCharArray(); + } + } + String environmentVariable = System.getenv(passwordEnvironmentVariable); + if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { + return environmentVariable.toCharArray(); + } + log.error("No password provided. Please provide your password."); + System.exit(2); + return null; // not reachable due to System.exit in previous line + } + + } + @Override + public String toString() { + return new StringJoiner(", ", SeekAuthenticationOptions.class.getSimpleName() + "[", "]") + .add("user='" + seekUser + "'") + .toString(); + //ATTENTION: do not expose the password here! + } +} \ No newline at end of file diff --git a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java index 6bed496..4ae0323 100644 --- a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java +++ b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java @@ -37,7 +37,7 @@ public class SpaceStatisticsCommand implements Runnable { names = {"--show-settings"}) private boolean allSpaces; @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); @Override public void run() { diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index dbaeb67..4160d45 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -1,9 +1,7 @@ package life.qbic.io.commandline; import ch.ethz.sis.openbis.generic.OpenBIS; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; @@ -17,15 +15,18 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.xml.parsers.ParserConfigurationException; import life.qbic.App; import life.qbic.model.AssayWithQueuedAssets; import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.download.SEEKConnector.SeekStructurePostRegistrationInformation; import life.qbic.model.isa.SeekStructure; import life.qbic.model.download.OpenbisConnector; import life.qbic.model.download.SEEKConnector; import life.qbic.model.download.SEEKConnector.AssetToUpload; import org.apache.commons.codec.binary.Base64; +import org.xml.sax.SAXException; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; @@ -36,7 +37,7 @@ "Transfers metadata and (optionally) data from openBIS to SEEK. Experiments, samples and " + "dataset information are always transferred together (as assays, samples and one of " + "several data types in SEEK). Dataset info always links back to the openBIS path of " - + "the respective dataset. The data itself will can transferred and stored in SEEK " + + "the respective dataset. The data itself can be transferred and stored in SEEK " + "using the '-d' flag." + "To completely exclude some dataset information from being transferred, a " + "file ('--blacklist') containing dataset codes (from openBIS) can be specified." @@ -47,6 +48,12 @@ public class TransferDataToSeekCommand implements Runnable { @Parameters(arity = "1", paramLabel = "openbis id", description = "The identifier of the " + "experiment, sample or dataset to transfer.") private String objectID; + @Parameters(arity = "1", paramLabel = "seek study", description = "Title of the study in SEEK" + + "where nodes should be added.") + private String studyTitle; + @Option(names = "--seek-project", description = "Title of the project in SEEK where nodes should" + + "be added. Can alternatively be provided via the config file as 'seek_default_project'.") + private String projectTitle; @Option(names = "--blacklist", description = "Path to file specifying by dataset " + "dataset code which openBIS datasets not to transfer to SEEK. The file must contain one code " + "per line.") @@ -55,28 +62,42 @@ public class TransferDataToSeekCommand implements Runnable { + "information in SEEK for the specified openBIS input should not be updated, but new nodes " + "created.") private boolean noUpdate; - /*@Option(names = {"-sn", "--seek-node"}, paramLabel = "seek node", description = - "The target node in SEEK to transfer to. Must correspond to " - + "the type of oopenBIS identifier chosen: experiment - assay; sample - sample; dataset - any of the data types. If no node is specified, " - + "a new data structure will be created in SEEK, starting from the related experiment.") - private String seekNode; - */ @Option(names = {"-d", "--data"}, description = "Transfers the data itself to SEEK along with the metadata. " + "Otherwise only the link(s) to the openBIS object will be created in SEEK.") private boolean transferData; @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + SeekAuthenticationOptions seekAuth = new SeekAuthenticationOptions(); + @Mixin + OpenbisAuthenticationOptions openbisAuth = new OpenbisAuthenticationOptions(); OpenbisConnector openbis; SEEKConnector seek; OpenbisSeekTranslator translator; + //500 MB - user will be informed that the transfer will take a while, for each file larger than this + private final long FILE_WARNING_SIZE = 500*1024*1024; @Override public void run() { + App.readConfig(); + System.out.printf("Transfer openBIS -> SEEK started.%n"); + System.out.printf("Provided openBIS object: %s%n", objectID); + System.out.printf("Provided SEEK study title: %s%n", studyTitle); + if(projectTitle!=null && !projectTitle.isBlank()) { + System.out.printf("Provided SEEK project title: %s%n", projectTitle); + } else { + System.out.printf("No SEEK project title provided, will search config file.%n"); + } + System.out.printf("Provided SEEK study title: %s%n", studyTitle); + System.out.printf("Transfer datasets to SEEK? %s%n", transferData); + System.out.printf("Update existing assay if found? %s%n", !noUpdate); + if(blacklistFile!=null && !blacklistFile.isBlank()) { + System.out.printf("File with datasets codes that won't be transferred: %s%n", blacklistFile); + } + System.out.println("Connecting to openBIS..."); - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), - auth.getOpenbisAS(), auth.getOpenbisDSS()); + OpenBIS authentication = App.loginToOpenBIS(openbisAuth.getOpenbisPassword(), + openbisAuth.getOpenbisUser(), openbisAuth.getOpenbisAS(), openbisAuth.getOpenbisDSS()); openbis = new OpenbisConnector(authentication); @@ -104,35 +125,22 @@ public void run() { System.out.println("Connecting to SEEK..."); byte[] httpCredentials = Base64.encodeBase64( - (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); + (seekAuth.getSeekUser() + ":" + new String(seekAuth.getSeekPassword())).getBytes()); try { - seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, auth.getOpenbisBaseURL(), - "seek_test", "lisym default study"); + String project = App.configProperties.get("seek_default_project"); + if(project == null || project.isBlank()) { + throw new RuntimeException("a default project must be provided via config "+ + "('seek_default_project') or parameter."); + } + seek = new SEEKConnector(seekAuth.getSeekURL(), httpCredentials, + openbisAuth.getOpenbisBaseURL(), App.configProperties.get("seek_default_project")); + seek.setDefaultStudy(studyTitle); translator = seek.getTranslator(); - } catch (URISyntaxException | IOException | InterruptedException e) { + } catch (URISyntaxException | IOException | InterruptedException | + ParserConfigurationException | SAXException e) { throw new RuntimeException(e); } - /* - if (seekNode != null) { - try { - if (!seek.endPointExists(seekNode)) { - System.out.println(seekNode + " could not be found"); - return; - } else { - if (isExperiment) { - Experiment experimentWithSamplesAndDatasets = openbis.getExperimentWithDescendants( - objectID); - //SeekStructure seekStructure = translator.translate(seekNode, experimentWithSamplesAndDatasets); - //seek.fillAssayWithSamplesAndDatasets(seekNode, seekStructure); - } - } - } catch (URISyntaxException | InterruptedException | IOException e) { - throw new RuntimeException(e); - } - } - - */ try { System.out.println("Collecting information from openBIS..."); OpenbisExperimentWithDescendants experiment = openbis.getExperimentWithDescendants(objectID); @@ -143,43 +151,24 @@ public void run() { System.out.println("Translating openBIS property codes to SEEK names..."); Map sampleTypesToIds = seek.getSampleTypeNamesToIDs(); System.out.println("Creating SEEK structure..."); - SeekStructure nodeWithChildren = translator.translate(experiment, sampleTypesToIds, blacklist); - if(assayID.isEmpty()) { - System.out.println("Creating new nodes..."); - createNewNodes(nodeWithChildren); + SeekStructure nodeWithChildren = translator.translate(experiment, sampleTypesToIds, blacklist, + transferData); + + if(assayID.isEmpty() || noUpdate) { + System.out.println("Creating new node(s)..."); + SeekStructurePostRegistrationInformation postRegInfo = createNewNodes(nodeWithChildren); + System.out.println("Creating links to SEEK objects in openBIS..."); + openbis.createSeekLinks(postRegInfo); } else { System.out.println("Updating nodes..."); - updateNodes(nodeWithChildren, assayID.get()); + SeekStructurePostRegistrationInformation postRegInfo = updateNodes(nodeWithChildren, + assayID.get()); + System.out.println("Updating links to SEEK objects in openBIS..."); + openbis.updateSeekLinks(postRegInfo); } } catch (URISyntaxException | InterruptedException | IOException e) { throw new RuntimeException(e); } - - /* - - List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); - - if(datasets.isEmpty()) { - System.out.println(datasetCode+" not found"); - return; - } - - DatasetWithProperties result = new DatasetWithProperties(datasets.get(0)); - Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", - result.getExperiment().getIdentifier()); - patientID.ifPresent(s -> result.addProperty("patientID", s)); - - System.out.println("Found dataset, downloading."); - System.out.println(); - - final String tmpPath = "tmp/"; - - File downloadFolder = openbis.downloadDataset(tmpPath, datasetCode); - - - - cleanupTemp(new File(tmpPath)); -*/ System.out.println("Done"); } @@ -206,23 +195,50 @@ private Set parseBlackList(String blacklistFile) { } } - private void updateNodes(SeekStructure nodeWithChildren, String assayID) { - String updatedEndpoint = seek.updateNode(nodeWithChildren, assayID, transferData); - System.out.printf("%s was successfully updated.%n", updatedEndpoint); + private SeekStructurePostRegistrationInformation updateNodes(SeekStructure nodeWithChildren, + String assayID) throws URISyntaxException, IOException, InterruptedException { + SeekStructurePostRegistrationInformation postRegInfo = seek.updateNode(nodeWithChildren, assayID); + System.out.printf("%s was successfully updated.%n", postRegInfo.getExperimentIDWithEndpoint().getRight()); + return postRegInfo; } - private void createNewNodes(SeekStructure nodeWithChildren) + private SeekStructurePostRegistrationInformation createNewNodes(SeekStructure nodeWithChildren) throws URISyntaxException, IOException, InterruptedException { - AssayWithQueuedAssets assetsOfAssayToUpload = seek.createNode(nodeWithChildren, transferData); + SeekStructurePostRegistrationInformation postRegistrationInformation = + seek.createNode(nodeWithChildren); + AssayWithQueuedAssets assetsOfAssayToUpload = postRegistrationInformation.getAssayWithQueuedAssets(); if(transferData) { + final String tmpFolderPath = "tmp/"; for(AssetToUpload asset : assetsOfAssayToUpload.getAssets()) { - System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); - String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), - () -> openbis.streamDataset(asset.getDataSetCode(), asset.getFilePath())); - System.out.printf("File stored here: %s%n", fileURL); + String filePath = asset.getFilePath(); + String dsCode = asset.getDataSetCode(); + if(asset.getFileSizeInBytes() > 1000*1024*1024) { + System.out.printf("Skipping %s due to size...%n", + filePath); + } else if (asset.getFileSizeInBytes() > 300 * 1024 * 1024) { + System.out.printf("File is %s MB...streaming might take a while%n", + asset.getFileSizeInBytes() / (1024 * 1024)); + System.out.printf("Downloading file %s from openBIS to tmp folder due to size...%n", + filePath); + File tmpFile = openbis.downloadDataset(tmpFolderPath, dsCode, filePath); + + System.out.printf("Uploading file to SEEK...%n"); + String fileURL = seek.uploadFileContent(asset.getBlobEndpoint(), tmpFile.getAbsolutePath()); + System.out.printf("File stored here: %s%n", fileURL); + } else { + System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); + + String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), + () -> openbis.streamDataset(asset.getDataSetCode(), asset.getFilePath())); + System.out.printf("File stored here: %s%n", fileURL); + } } + System.out.printf("Cleaning up temp folder%n"); + cleanupTemp(new File(tmpFolderPath)); } + System.out.printf("%s was successfully created.%n", assetsOfAssayToUpload.getAssayEndpoint()); + return postRegistrationInformation; } private boolean sampleExists(String objectID) { @@ -246,6 +262,7 @@ private Optional getAssayIDForOpenBISExperiment(Experiment experiment) String permID = experiment.getPermId().getPermId(); List assayIDs = seek.searchAssaysContainingKeyword(permID); if(assayIDs.isEmpty()) { + System.err.println("no assay found containing "+permID); return Optional.empty(); } if(assayIDs.size() == 1) { @@ -254,26 +271,6 @@ private Optional getAssayIDForOpenBISExperiment(Experiment experiment) throw new RuntimeException("Experiment identifier "+permID+ " was found in more than one assay: "+assayIDs); } - private void sendDatasetToSeek(String datasetCode, String assayID) - throws URISyntaxException, IOException, InterruptedException { - assayID = "3"; - System.out.println("Searching dataset in openBIS..."); - List datasets = openbis.findDataSets( - Arrays.asList(datasetCode)); - if(datasets.isEmpty()) { - return; - } - DataSet dataset = datasets.get(0); - List files = openbis.getDatasetFiles(dataset); - List assets = seek.createAssets(files, Arrays.asList(assayID)); - for(AssetToUpload asset : assets) { - System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); - String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), - () -> openbis.streamDataset(datasetCode, asset.getFilePath())); - System.out.printf("File stored here: %s%n", fileURL); - } - } - private void cleanupTemp(File tmpFolder) { File[] files = tmpFolder.listFiles(); if (files != null) { //some JVMs return null for empty dirs diff --git a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java index 301f09a..3ba3d55 100644 --- a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java @@ -4,21 +4,31 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; import java.io.IOException; import java.net.URISyntaxException; +import javax.xml.parsers.ParserConfigurationException; import life.qbic.App; +import life.qbic.io.PropertyReader; import life.qbic.model.OpenbisSeekTranslator; import life.qbic.model.SampleTypesAndMaterials; import life.qbic.model.download.OpenbisConnector; import life.qbic.model.download.SEEKConnector; import org.apache.commons.codec.binary.Base64; +import org.xml.sax.SAXException; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; @Command(name = "sample-type-transfer", description = "Transfers sample types from openBIS to SEEK.") public class TransferSampleTypesToSeekCommand implements Runnable { @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + SeekAuthenticationOptions seekAuth = new SeekAuthenticationOptions(); + @Mixin + OpenbisAuthenticationOptions openbisAuth = new OpenbisAuthenticationOptions(); + @Option(names = "--ignore-existing", description = "Use to specify that existing " + + "sample-types of the same name in SEEK should be ignored and the sample-type created a " + + "second time.") + private boolean ignoreExisting; OpenbisConnector openbis; SEEKConnector seek; OpenbisSeekTranslator translator; @@ -27,19 +37,25 @@ public class TransferSampleTypesToSeekCommand implements Runnable { public void run() { System.out.println("auth..."); - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), - auth.getOpenbisAS(), auth.getOpenbisDSS()); + OpenBIS authentication = App.loginToOpenBIS(openbisAuth.getOpenbisPassword(), + openbisAuth.getOpenbisUser(), openbisAuth.getOpenbisAS(), openbisAuth.getOpenbisDSS()); System.out.println("openbis..."); openbis = new OpenbisConnector(authentication); byte[] httpCredentials = Base64.encodeBase64( - (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); + (seekAuth.getSeekUser() + ":" + new String(seekAuth.getSeekPassword())).getBytes()); try { - seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, auth.getOpenbisBaseURL(), - "seek_test", "lisym default study"); + String project = App.configProperties.get("seek_default_project"); + if(project == null || project.isBlank()) { + throw new RuntimeException("a default project must be provided via config "+ + "('seek_default_project') or parameter."); + } + seek = new SEEKConnector(seekAuth.getSeekURL(), httpCredentials, openbisAuth.getOpenbisBaseURL(), + App.configProperties.get("seek_default_project")); translator = seek.getTranslator(); - } catch (URISyntaxException | IOException | InterruptedException e) { + } catch (URISyntaxException | IOException | InterruptedException | + ParserConfigurationException | SAXException e) { throw new RuntimeException(e); } @@ -48,8 +64,13 @@ public void run() { try { for(SampleType type : types.getSampleTypes()) { System.err.println("creating "+type.getCode()); - String sampleTypeId = seek.createSampleType(translator.translate(type)); - System.err.println("created "+sampleTypeId); + if(!ignoreExisting && seek.sampleTypeExists(type.getCode())) { + System.err.println(type.getCode()+ " is already in SEEK. If you want to create a new " + + "sample type using the same name, you can use the --ignore-existing flag."); + } else { + String sampleTypeId = seek.createSampleType(translator.translate(type)); + System.err.println("created "+sampleTypeId); + } } } catch (URISyntaxException | IOException | InterruptedException e) { throw new RuntimeException(e); diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index a24a0c7..98e9f11 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -26,7 +26,7 @@ public class UploadDatasetCommand implements Runnable { + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) private List parents = new ArrayList<>(); @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); private OpenbisConnector openbis; diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 8e36b1e..80d40bc 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -26,7 +26,7 @@ public class UploadPetabResultCommand implements Runnable { // + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) private List parents = new ArrayList<>(); @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); private OpenbisConnector openbis; private PetabParser petabParser = new PetabParser(); @@ -63,7 +63,6 @@ public void run() { } } System.out.println("Uploading dataset..."); - //TODO copy and remove source references here DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), experimentID, parents); System.out.printf("Dataset %s was successfully created%n", result.getPermId()); } diff --git a/src/main/java/life/qbic/model/AssetInformation.java b/src/main/java/life/qbic/model/AssetInformation.java new file mode 100644 index 0000000..476d65e --- /dev/null +++ b/src/main/java/life/qbic/model/AssetInformation.java @@ -0,0 +1,41 @@ +package life.qbic.model; + +public class AssetInformation { + + private String seekID; + private String title; + private String description; + private String assetType; + private String openbisPermId; + + public AssetInformation(String assetID, String assetType, String title, String description) { + this.seekID = assetID; + this.title = title; + this.description = description; + this.assetType = assetType; + } + + public String getAssetType() { + return assetType; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public String getSeekID() { + return seekID; + } + + public void setOpenbisPermId(String id) { + this.openbisPermId = id; + } + + public String getOpenbisPermId() { + return openbisPermId; + } +} diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index dfcc025..99fb500 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -9,9 +9,9 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.io.File; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -20,6 +20,11 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import life.qbic.App; +import life.qbic.io.PropertyReader; import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.ISAAssay; import life.qbic.model.isa.ISASample; @@ -27,18 +32,76 @@ import life.qbic.model.isa.ISASampleType.SampleAttribute; import life.qbic.model.isa.ISASampleType.SampleAttributeType; import life.qbic.model.isa.SeekStructure; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; public class OpenbisSeekTranslator { private final String DEFAULT_PROJECT_ID; - private final String DEFAULT_STUDY_ID; - private final String DEFAULT_TRANSFERRED_SAMPLE_TITLE = "openBIS Name"; + private String INVESTIGATION_ID; + private String STUDY_ID; private final String openBISBaseURL; + private Map experimentTypeToAssayClass; + private Map dataTypeToAttributeType; + private Map datasetTypeToAssetType; + private Map experimentTypeToAssayType; - public OpenbisSeekTranslator(String openBISBaseURL, String defaultProjectID, String defaultStudyID) { + public OpenbisSeekTranslator(String openBISBaseURL, String defaultProjectID) + throws IOException, ParserConfigurationException, SAXException { this.openBISBaseURL = openBISBaseURL; this.DEFAULT_PROJECT_ID = defaultProjectID; - this.DEFAULT_STUDY_ID = defaultStudyID; + parseConfigs(); + if(!App.configProperties.containsKey("seek_openbis_sample_title")) { + throw new RuntimeException("Script can not be run, since 'seek_openbis_sample_title' was not " + + "provided."); + } + } + + /** + * Parses mandatory mapping information from mandatory config files. Other files may be added. + */ + private void parseConfigs() throws IOException, ParserConfigurationException, SAXException { + final String dataTypeToAttributeType = "openbis_datatype_to_seek_attributetype.xml"; + final String datasetToAssaytype = "dataset_type_to_assaytype.properties"; + final String experimentTypeToAssayClass = "experiment_type_to_assay_class.properties"; + final String experimentTypeToAssayType = "more.properties"; + this.experimentTypeToAssayType = PropertyReader.getProperties(experimentTypeToAssayType); + this.datasetTypeToAssetType = PropertyReader.getProperties(datasetToAssaytype); + this.experimentTypeToAssayClass = PropertyReader.getProperties(experimentTypeToAssayClass); + this.dataTypeToAttributeType = parseAttributeXML(dataTypeToAttributeType); + } + + private Map parseAttributeXML(String dataTypeToAttributeType) + throws ParserConfigurationException, IOException, SAXException { + Map result = new HashMap<>(); + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(dataTypeToAttributeType); + NodeList elements = document.getElementsByTagName("entry"); + for (int i = 0; i < elements.getLength(); i++) { + Node node = elements.item(i); + DataType openbisType = DataType.valueOf(node.getAttributes() + .getNamedItem("type") + .getNodeValue()); + NodeList nodes = node.getChildNodes(); + String seekId = "", seekTitle = "", seekBaseType = ""; + for (int j = 0; j < nodes.getLength(); j++) { + Node n = nodes.item(j); + if (n.getNodeName().equals("seek_id")) { + seekId = n.getTextContent(); + } + if (n.getNodeName().equals("seek_type")) { + seekBaseType = n.getTextContent(); + } + if (n.getNodeName().equals("seek_title")) { + seekTitle = n.getTextContent(); + } + } + result.put(openbisType, new SampleAttributeType(seekId, seekTitle, seekBaseType)); + } + return result; } private String generateOpenBISLinkFromPermID(String entityType, String permID) { @@ -51,44 +114,9 @@ private String generateOpenBISLinkFromPermID(String entityType, String permID) { return builder.toString(); } - Map experimentTypeToAssayClass = Map.ofEntries( - entry("00_MOUSE_DATABASE", "EXP"), - entry("00_PATIENT_DATABASE", "EXP"), - entry("00_STANDARD_OPERATING_PROTOCOLS", "EXP"), - entry("01_BIOLOGICAL_EXPERIMENT", "EXP"), - entry("02_MASSSPECTROMETRY_EXPERIMENT", "EXP"),//where is petab modeling attached? - entry("03_HISTOLOGICAL_ANALYSIS", "EXP"), - entry("04_MICRO_CT", "EXP") - ); - - Map dataTypeToAttributeType = Map.ofEntries( - entry(DataType.INTEGER, new SampleAttributeType("4", "Integer", "Integer")), - entry(DataType.VARCHAR, new SampleAttributeType("8", "String", "String")), - entry(DataType.MULTILINE_VARCHAR, new SampleAttributeType("7", "Text", "Text")), - entry(DataType.REAL, new SampleAttributeType("3", "Real number", "Float")), - entry(DataType.TIMESTAMP, new SampleAttributeType("1", "Date time", "DateTime")), - entry(DataType.BOOLEAN, new SampleAttributeType("16", "Boolean", "Boolean")), - entry(DataType.CONTROLLEDVOCABULARY, //we use String for now - new SampleAttributeType("8", "String", "String")), - entry(DataType.MATERIAL, //not used anymore in this form - new SampleAttributeType("8", "String", "String")), - entry(DataType.HYPERLINK, new SampleAttributeType("8", "String", "String")), - entry(DataType.XML, new SampleAttributeType("7", "Text", "Text")), - entry(DataType.SAMPLE, //we link the sample as URL to openBIS for now - new SampleAttributeType("5", "Web link", "String")), - entry(DataType.DATE, new SampleAttributeType("2", "Date time", "Date")) - ); - - Map experimentTypeToAssayType = Map.ofEntries( - entry("00_MOUSE_DATABASE", ""), - entry("00_PATIENT_DATABASE", ""), - entry("00_STANDARD_OPERATING_PROTOCOLS", ""), - entry("01_BIOLOGICAL_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Cultivation_experiment"), - entry("02_MASSSPECTROMETRY_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Proteomics"), - entry("03_HISTOLOGICAL_ANALYSIS", ""), - entry("04_MICRO_CT", "") - ); - + /** + * not mandatory, but nice to have? + */ Map fileExtensionToDataFormat = Map.ofEntries( entry("fastq.gz", "http://edamontology.org/format_1930"), entry("fastq", "http://edamontology.org/format_1930"), @@ -96,34 +124,13 @@ private String generateOpenBISLinkFromPermID(String entityType, String permID) { entry("yaml", "http://edamontology.org/format_3750"), entry("raw", "http://edamontology.org/format_3712"), entry("tsv", "http://edamontology.org/format_3475"), - entry("csv", "http://edamontology.org/format_3752") + entry("csv", "http://edamontology.org/format_3752"), + entry("txt", "http://edamontology.org/format_2330") ); - Map datasetTypeToAssetType = Map.ofEntries( - entry("ANALYSIS_NOTEBOOK", "documents"), - entry("ANALYZED_DATA", "data_files"), - entry("ATTACHMENT", "documents"), - entry("ELN_PREVIEW", ""), - entry("EXPERIMENT_PROTOCOL", "sops"), - entry("EXPERIMENT_RESULT", "documents"), - entry("HISTOLOGICAL_SLIDE", "data_files"), - entry("IB_DATA", "data_files"), - entry("LUMINEX_DATA", "data_files"), - entry("MS_DATA_ANALYZED", "data_files"), - entry("MS_DATA_RAW", "data_files"), - entry("OTHER_DATA", "data_files"), - entry("PROCESSED_DATA", "data_files"), - entry("PUBLICATION_DATA", "publications"), - entry("QPCR_DATA", "data_files"), - entry("RAW_DATA", "data_files"), - entry("SOURCE_CODE", "documents"), - entry("TEST_CONT", ""), - entry("TEST_DAT", ""), - entry("UNKNOWN", "data_files") - ); - public ISASampleType translate(SampleType sampleType) { - SampleAttribute titleAttribute = new SampleAttribute(DEFAULT_TRANSFERRED_SAMPLE_TITLE, + SampleAttribute titleAttribute = new SampleAttribute( + App.configProperties.get("seek_openbis_sample_title"), dataTypeToAttributeType.get(DataType.VARCHAR), true, false); ISASampleType type = new ISASampleType(sampleType.getCode(), titleAttribute, DEFAULT_PROJECT_ID); @@ -147,15 +154,17 @@ public String dataFormatAnnotationForExtension(String fileExtension) { } public SeekStructure translate(OpenbisExperimentWithDescendants experiment, - Map sampleTypesToIds, Set blacklist) throws URISyntaxException { + Map sampleTypesToIds, Set blacklist, boolean transferData) + throws URISyntaxException { Experiment exp = experiment.getExperiment(); String expType = exp.getType().getCode(); String title = exp.getCode()+" ("+exp.getPermId().getPermId()+")"; - ISAAssay assay = new ISAAssay(title, DEFAULT_STUDY_ID, experimentTypeToAssayClass.get(expType), - new URI(experimentTypeToAssayType.get(expType)));//TODO + ISAAssay assay = new ISAAssay(title, STUDY_ID, experimentTypeToAssayClass.get(expType), + new URI(experimentTypeToAssayType.get(expType))); + + SeekStructure result = new SeekStructure(assay, exp.getIdentifier().getIdentifier()); - List samples = new ArrayList<>(); for(Sample sample : experiment.getSamples()) { SampleType sampleType = sample.getType(); @@ -180,26 +189,26 @@ public SeekStructure translate(OpenbisExperimentWithDescendants experiment, attributes.put(typeCodesToNames.get(code), value); } - attributes.put(DEFAULT_TRANSFERRED_SAMPLE_TITLE, sample.getIdentifier().getIdentifier()); + String sampleID = sample.getIdentifier().getIdentifier(); + attributes.put(App.configProperties.get("seek_openbis_sample_title"), sampleID); String sampleTypeId = sampleTypesToIds.get(sampleType.getCode()); - ISASample isaSample = new ISASample(sample.getPermId().getPermId(), attributes, sampleTypeId, - Collections.singletonList(DEFAULT_PROJECT_ID)); - samples.add(isaSample); + ISASample isaSample = new ISASample(sample.getIdentifier().getIdentifier(), attributes, + sampleTypeId, Collections.singletonList(DEFAULT_PROJECT_ID)); + result.addSample(isaSample, sampleID); } - Map isaToOpenBISFile = new HashMap<>(); - //create ISA files for assets. If actual data is uploaded is determined later based upon flag + //create ISA files for assets. If actual data is to be uploaded is determined later based on flag for(DatasetWithProperties dataset : experiment.getDatasets()) { String permID = dataset.getCode(); if(!blacklist.contains(permID)) { for(DataSetFile file : experiment.getFilesForDataset(permID)) { String datasetType = getDatasetTypeOfFile(file, experiment.getDatasets()); - datasetFileToSeekAsset(file, datasetType) - .ifPresent(seekAsset -> isaToOpenBISFile.put(seekAsset, file)); + datasetFileToSeekAsset(file, datasetType, transferData) + .ifPresent(seekAsset -> result.addAsset(seekAsset, file)); } } } - return new SeekStructure(assay, samples, isaToOpenBISFile); + return result; } private String getDatasetTypeOfFile(DataSetFile file, List dataSets) { @@ -217,14 +226,20 @@ private String getDatasetTypeOfFile(DataSetFile file, List datasetFileToSeekAsset(DataSetFile file, String datasetType) { + private Optional datasetFileToSeekAsset(DataSetFile file, String datasetType, + boolean transferData) { if (!file.getPath().isBlank() && !file.isDirectory()) { File f = new File(file.getPath()); - String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode + ": " + f.getName(); + String datasetPermID = file.getDataSetPermId().toString(); + String assetName = datasetPermID + ": " + f.getName(); String assetType = assetForDatasetType(datasetType); GenericSeekAsset isaFile = new GenericSeekAsset(assetType, assetName, file.getPath(), - Arrays.asList(DEFAULT_PROJECT_ID)); + Arrays.asList(DEFAULT_PROJECT_ID), file.getFileLength()); + //reference the openbis dataset in the description - if transferData flag is false, this will + //also add a second link instead of the (non-functional) download link to a non-existent blob. + //it seems that directly linking to files needs an open session, so we only set the dataset for now + String datasetLink = generateOpenBISLinkFromPermID("DATA_SET", datasetPermID); + isaFile.setDatasetLink(datasetLink, transferData); String fileExtension = f.getName().substring(f.getName().lastIndexOf(".") + 1); String annotation = dataFormatAnnotationForExtension(fileExtension); if (annotation != null) { @@ -235,4 +250,11 @@ private Optional datasetFileToSeekAsset(DataSetFile file, Stri return Optional.empty(); } + public void setDefaultStudy(String studyID) { + this.STUDY_ID = studyID; + } + + public void setDefaultInvestigation(String investigationID) { + this.INVESTIGATION_ID = investigationID; + } } diff --git a/src/main/java/life/qbic/model/SampleInformation.java b/src/main/java/life/qbic/model/SampleInformation.java new file mode 100644 index 0000000..73a264b --- /dev/null +++ b/src/main/java/life/qbic/model/SampleInformation.java @@ -0,0 +1,28 @@ +package life.qbic.model; + +import java.util.Map; + +public class SampleInformation { + + private String seekID; + private String openBisIdentifier; + private Map attributes; + + public SampleInformation(String sampleID, String title, Map attributesMap) { + this.seekID = sampleID; + this.openBisIdentifier = title; + this.attributes = attributesMap; + } + + public String getSeekID() { + return seekID; + } + + public Map getAttributes() { + return attributes; + } + + public String getOpenBisIdentifier() { + return openBisIdentifier; + } +} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index 7e5d7ed..386e494 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -6,21 +6,22 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.search.DataSetSearchCriteria; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.update.DataSetUpdate; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.EntityKind; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.search.ExperimentSearchCriteria; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.PropertyAssignment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.update.ExperimentUpdate; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleTypeFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SamplePermId; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleTypeSearchCriteria; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.update.SampleUpdate; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria; @@ -46,13 +47,15 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import life.qbic.App; import life.qbic.model.DatasetWithProperties; import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.SampleTypeConnection; import life.qbic.model.SampleTypesAndMaterials; +import life.qbic.model.download.SEEKConnector.SeekStructurePostRegistrationInformation; +import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -61,9 +64,10 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); private final OpenBIS openBIS; - //public final static Pattern experimentIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); - //public final static Pattern sampleIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); - public static Pattern datasetCodePattern = Pattern.compile("[0-9]^{17}-[0-9]+"); + public static Pattern datasetCodePattern = Pattern.compile("[0-9]{17}-[0-9]+"); + public final String EXPERIMENT_LINK_PROPERTY = "EXPERIMENT_NAME"; + public final String SAMPLE_LINK_PROPERTY = "experimentLink"; + public final String DATASET_LINK_PROPERTY = "experimentLink"; public OpenbisConnector(OpenBIS authentication) { this.openBIS = authentication; @@ -139,10 +143,10 @@ public List listDatasetsOfExperiment(List spaces, String experi return openBIS.searchDataSets(criteria, options).getObjects(); } - public File downloadDataset(String targetPath, String datasetID) { + public File downloadDataset(String targetPath, String datasetID, String filePath) { DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetID), - ""); + filePath); // Setting recursive flag to true will return both subfolders and files options.setRecursive(true); @@ -171,7 +175,7 @@ public File downloadDataset(String targetPath, String datasetID) { } } } - return new File(targetPath); + return new File(targetPath, filePath.replace("original/","")); } public InputStream streamDataset(String datasetCode, String filePath) { @@ -232,20 +236,22 @@ public Map queryFullSampleHierarchy(List return hierarchy; } - private Optional getPropertyFromSampleHierarchy(String propertyName, List samples) { + private Set getPropertiesFromSampleHierarchy(String propertyName, List samples, + Set foundProperties) { for(Sample s : samples) { if(s.getProperties().containsKey(propertyName)) { - return Optional.of(s.getProperties().get(propertyName)); + foundProperties.add(s.getProperties().get(propertyName)); + return foundProperties; } - return getPropertyFromSampleHierarchy(propertyName, s.getParents()); + return getPropertiesFromSampleHierarchy(propertyName, s.getParents(), foundProperties); } - return Optional.empty(); + return foundProperties; } - public Optional findPropertyInSampleHierarchy(String propertyName, + public Set findPropertiesInSampleHierarchy(String propertyName, ExperimentIdentifier experimentId) { - return getPropertyFromSampleHierarchy(propertyName, - getSamplesWithAncestorsOfExperiment(experimentId)); + return getPropertiesFromSampleHierarchy(propertyName, + getSamplesWithAncestorsOfExperiment(experimentId), new HashSet<>()); } public Map> getExperimentsBySpace(List spaces) { @@ -377,18 +383,22 @@ private String getSpaceFromSampleOrExperiment(DataSet d) { } private List getSamplesWithAncestorsOfExperiment(ExperimentIdentifier experimentId) { - SampleFetchOptions allProps = new SampleFetchOptions(); - allProps.withType(); - allProps.withProperties(); - SampleFetchOptions withAncestors = new SampleFetchOptions(); - withAncestors.withParentsUsing(allProps); - withAncestors.withProperties(); - withAncestors.withType(); + int numberOfFetchedLevels = 10; + SampleFetchOptions previousLevel = null; + for(int i = 0; i < numberOfFetchedLevels; i++) { + SampleFetchOptions withAncestors = new SampleFetchOptions(); + withAncestors.withProperties(); + withAncestors.withType(); + if (previousLevel != null) { + withAncestors.withParentsUsing(previousLevel); + } + previousLevel = withAncestors; + } SampleSearchCriteria criteria = new SampleSearchCriteria(); criteria.withExperiment().withId().thatEquals(experimentId); - return openBIS.searchSamples(criteria, withAncestors).getObjects(); + return openBIS.searchSamples(criteria, previousLevel).getObjects(); } public List findDataSets(List codes) { @@ -491,4 +501,86 @@ public SampleTypesAndMaterials getSampleTypesWithMaterials() { } return new SampleTypesAndMaterials(sampleTypes, sampleTypesAsMaterials); } + + public void createSeekLinks(SeekStructurePostRegistrationInformation postRegistrationInformation) { + Pair experimentInfo = postRegistrationInformation.getExperimentIDWithEndpoint(); + ExperimentIdentifier id = new ExperimentIdentifier(experimentInfo.getLeft()); + String endpoint = experimentInfo.getRight(); + Map props = new HashMap<>(); + props.put(EXPERIMENT_LINK_PROPERTY, endpoint); + updateExperimentProperties(id, props, false); + } + + public void updateSeekLinks(SeekStructurePostRegistrationInformation postRegistrationInformation) { + } + + private void updateExperimentProperties(ExperimentIdentifier id, Map properties, + boolean overwrite) { + ExperimentUpdate update = new ExperimentUpdate(); + update.setExperimentId(id); + if(overwrite) { + update.setProperties(properties); + } else { + ExperimentFetchOptions options = new ExperimentFetchOptions(); + options.withProperties(); + Experiment oldExp = openBIS.getExperiments(Arrays.asList(id), options).get(id); + for(String property : properties.keySet()) { + String newValue = properties.get(property); + String oldValue = oldExp.getProperty(property); + if(oldValue == null || oldValue.isEmpty() || oldValue.equals(newValue)) { + update.setProperty(property, newValue); + } else if(!newValue.isBlank()) { + update.setProperty(property, oldValue+", "+newValue);//TODO this can be changed to any other strategy + } + } + } + openBIS.updateExperiments(Arrays.asList(update)); + } + + private void updateSampleProperties(SampleIdentifier id, Map properties, + boolean overwrite) { + SampleUpdate update = new SampleUpdate(); + update.setSampleId(id); + if(overwrite) { + update.setProperties(properties); + } else { + SampleFetchOptions options = new SampleFetchOptions(); + options.withProperties(); + Sample oldSample = openBIS.getSamples(Arrays.asList(id), options).get(id); + for(String property : properties.keySet()) { + String newValue = properties.get(property); + String oldValue = oldSample.getProperty(property); + if(oldValue == null || oldValue.isEmpty() || oldValue.equals(newValue)) { + update.setProperty(property, newValue); + } else { + update.setProperty(property, oldValue+", "+newValue);//TODO this can be changed to any other strategy + } + } + } + openBIS.updateSamples(Arrays.asList(update)); + } + + private void updateDatasetProperties(DataSetPermId id, Map properties, + boolean overwrite) { + DataSetUpdate update = new DataSetUpdate(); + update.setDataSetId(id); + if(overwrite) { + update.setProperties(properties); + } else { + DataSetFetchOptions options = new DataSetFetchOptions(); + options.withProperties(); + DataSet oldDataset = openBIS.getDataSets(Arrays.asList(id), options).get(id); + for(String property : properties.keySet()) { + String newValue = properties.get(property); + String oldValue = oldDataset.getProperty(property); + if(oldValue == null || oldValue.isEmpty() || oldValue.equals(newValue)) { + update.setProperty(property, newValue); + } else { + update.setProperty(property, oldValue+", "+newValue);//TODO this can be changed to any other strategy + } + } + } + openBIS.updateDataSets(Arrays.asList(update)); + } + } diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java index 16fb0ec..6214c7e 100644 --- a/src/main/java/life/qbic/model/download/SEEKConnector.java +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -16,23 +16,33 @@ import java.net.http.HttpResponse.BodyHandlers; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Supplier; +import java.util.regex.Matcher; +import javax.xml.parsers.ParserConfigurationException; import life.qbic.model.AssayWithQueuedAssets; +import life.qbic.model.AssetInformation; import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.SampleInformation; import life.qbic.model.isa.SeekStructure; import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.ISAAssay; import life.qbic.model.isa.ISASample; import life.qbic.model.isa.ISASampleType; import life.qbic.model.isa.ISAStudy; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.http.client.utils.URIBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.xml.sax.SAXException; public class SEEKConnector { @@ -42,20 +52,29 @@ public class SEEKConnector { private OpenbisSeekTranslator translator; private final String DEFAULT_PROJECT_ID; - public SEEKConnector(String apiURL, byte[] httpCredentials, String openBISBaseURL, - String defaultProjectTitle, String defaultStudyTitle) throws URISyntaxException, IOException, - InterruptedException { - this.apiURL = apiURL; + public SEEKConnector(String seekURL, byte[] httpCredentials, String openBISBaseURL, + String defaultProjectTitle) throws URISyntaxException, IOException, + InterruptedException, ParserConfigurationException, SAXException { + this.apiURL = seekURL; this.credentials = httpCredentials; Optional projectID = getProjectWithTitle(defaultProjectTitle); - if(projectID.isEmpty()) { - throw new RuntimeException("Failed to find project with title: " + defaultProjectTitle+". " + if (projectID.isEmpty()) { + throw new RuntimeException("Failed to find project with title: " + defaultProjectTitle + ". " + "Please provide an existing default project."); } DEFAULT_PROJECT_ID = projectID.get(); + translator = new OpenbisSeekTranslator(openBISBaseURL, DEFAULT_PROJECT_ID); + } + + public void setDefaultInvestigation(String investigationTitle) + throws URISyntaxException, IOException, InterruptedException { + translator.setDefaultInvestigation(searchNodeWithTitle("investigations", + investigationTitle)); + } - translator = new OpenbisSeekTranslator(openBISBaseURL, DEFAULT_PROJECT_ID, - searchNodeWithTitle("studies", defaultStudyTitle)); + public void setDefaultStudy(String studyTitle) + throws URISyntaxException, IOException, InterruptedException { + translator.setDefaultStudy(searchNodeWithTitle("studies", studyTitle)); } /** @@ -92,6 +111,23 @@ private Optional getProjectWithTitle(String projectTitle) return Optional.empty(); } + public String addStudy(ISAStudy assay) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/studies"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, assay.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + public String addAssay(ISAAssay assay) throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/assays"; @@ -126,6 +162,15 @@ public String createStudy(ISAStudy study) return idNode.asText(); } + private HttpRequest buildAuthorizedPATCHRequest(String endpoint, String body) throws URISyntaxException { + return HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .method("PATCH", HttpRequest.BodyPublishers.ofString(body)).build(); + } + private HttpRequest buildAuthorizedPOSTRequest(String endpoint, String body) throws URISyntaxException { return HttpRequest.newBuilder() .uri(new URI(endpoint)) @@ -160,13 +205,9 @@ public void printAttributeTypes() throws URISyntaxException, IOException, Interr .send(request, BodyHandlers.ofString()); System.err.println(response.body()); } + /* - -datatype by extension --assay equals experiment --investigation: pre-created in SEEK? --study: project -patient id should be linked somehow, maybe gender? --flexible object type to sample type? */ public void deleteSampleType(String id) throws URISyntaxException, IOException, @@ -188,8 +229,8 @@ public void deleteSampleType(String id) throws URISyntaxException, IOException, } } - public String createSampleType(ISASampleType sampleType) throws URISyntaxException, IOException, - InterruptedException { + public String createSampleType(ISASampleType sampleType) + throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/sample_types"; HttpResponse response = HttpClient.newBuilder().build() @@ -206,6 +247,25 @@ public String createSampleType(ISASampleType sampleType) throws URISyntaxExcepti return idNode.asText(); } + public String updateSample(ISASample isaSample, String sampleID) throws URISyntaxException, IOException, + InterruptedException { + String endpoint = apiURL+"/samples/"+sampleID; + isaSample.setSampleID(sampleID); + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPATCHRequest(endpoint, isaSample.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return endpoint+"/"+idNode.asText(); + } + public String createSample(ISASample isaSample) throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/samples"; @@ -221,10 +281,10 @@ public String createSample(ISASample isaSample) throws URISyntaxException, IOExc JsonNode rootNode = new ObjectMapper().readTree(response.body()); JsonNode idNode = rootNode.path("data").path("id"); - return idNode.asText(); + return endpoint+"/"+idNode.asText(); } - private AssetToUpload createDataFileAsset(String datasetCode, GenericSeekAsset data) + private AssetToUpload createAsset(String datasetCode, GenericSeekAsset data) throws IOException, URISyntaxException, InterruptedException { String endpoint = apiURL+"/"+data.getType(); @@ -242,16 +302,14 @@ private AssetToUpload createDataFileAsset(String datasetCode, GenericSeekAsset d .path("attributes") .path("content_blobs") .path(0).path("link"); - return new AssetToUpload(idNode.asText(), data.getFileName(), datasetCode); + return new AssetToUpload(idNode.asText(), data.getFileName(), datasetCode, data.fileSizeInBytes()); } - @Deprecated - private void uploadFileContent(String assetType, String assetID, String blobID, String file) + public String uploadFileContent(String blobEndpoint, String file) throws URISyntaxException, IOException, InterruptedException { - String endpoint = apiURL+"/"+assetType+"/"+assetID+"/content_blobs/"+blobID; HttpRequest request = HttpRequest.newBuilder() - .uri(new URI(endpoint)) + .uri(new URI(blobEndpoint)) .headers("Content-Type", "application/octet-stream") .headers("Accept", "application/octet-stream") .headers("Authorization", "Basic " + new String(credentials)) @@ -264,6 +322,7 @@ private void uploadFileContent(String assetType, String assetID, String blobID, System.err.println(response.body()); throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } + return blobEndpointToAssetURL(blobEndpoint); } public String uploadStreamContent(String blobEndpoint, @@ -287,11 +346,14 @@ public String uploadStreamContent(String blobEndpoint, System.err.println(response.body()); throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } else { - String fileURL = blobEndpoint.split("content_blobs")[0]; - return fileURL; + return blobEndpointToAssetURL(blobEndpoint); } } + private String blobEndpointToAssetURL(String blobEndpoint) { + return blobEndpoint.split("content_blobs")[0]; + } + public boolean endPointExists(String endpoint) throws URISyntaxException, IOException, InterruptedException { HttpRequest request = HttpRequest.newBuilder() @@ -305,59 +367,6 @@ public boolean endPointExists(String endpoint) return response.statusCode() == 200; } - /** - * Creates an optional asset for a file from an openBIS dataset. Folders are ignored. - * @param file - * @param assetType - * @param assays - * @return - * @throws IOException - * @throws URISyntaxException - * @throws InterruptedException - */ - public Optional createAssetForFile(DataSetFile file, String assetType, - List assays) - throws IOException, URISyntaxException, InterruptedException { - if(!file.getPath().isBlank() && !file.isDirectory()) { - File f = new File(file.getPath()); - String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode+": "+f.getName();//TODO what do we want to call the asset? - GenericSeekAsset isaFile = new GenericSeekAsset(assetType, assetName, file.getPath(), - Arrays.asList(DEFAULT_PROJECT_ID)); - isaFile.withAssays(assays); - String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); - String annotation = translator.dataFormatAnnotationForExtension(fileExtension); - if(annotation!=null) { - isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); - } - return Optional.of(createDataFileAsset(datasetCode, isaFile)); - } - return Optional.empty(); - } - - public List createAssets(List filesInDataset, - List assays) - throws IOException, URISyntaxException, InterruptedException { - List result = new ArrayList<>(); - for(DataSetFile file : filesInDataset) { - if(!file.getPath().isBlank() && !file.isDirectory()) { - File f = new File(file.getPath()); - String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode+": "+f.getName();//TODO what do we want to call the asset? - GenericSeekAsset isaFile = new GenericSeekAsset("data_files", assetName, file.getPath(), - Arrays.asList(DEFAULT_PROJECT_ID)); - isaFile.withAssays(assays); - String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); - String annotation = translator.dataFormatAnnotationForExtension(fileExtension); - if(annotation!=null) { - isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); - } - result.add(createDataFileAsset(datasetCode, isaFile)); - } - } - return result; - } - /** * Creates * @param isaToOpenBISFile @@ -373,7 +382,7 @@ public List createAssetsForAssays(Map result = new ArrayList<>(); for (GenericSeekAsset isaFile : isaToOpenBISFile.keySet()) { isaFile.withAssays(assays); - result.add(createDataFileAsset(isaToOpenBISFile.get(isaFile).getDataSetPermId().getPermId(), + result.add(createAsset(isaToOpenBISFile.get(isaFile).getDataSetPermId().getPermId(), isaFile)); } return result; @@ -427,11 +436,30 @@ private Map parseSampleTypesJSON(String json) throws JsonProcess return typesToIDs; } - private String searchNodeWithTitle(String nodeType, String title) + public boolean sampleTypeExists(String typeCode) + throws URISyntaxException, IOException, InterruptedException { + JsonNode result = genericSearch("sample_types", typeCode); + JsonNode hits = result.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + if (hit.get("attributes").get("title").asText().equals(typeCode)) { + return true; + } + } + return false; + } + + /** + * Performs a generic search and returns the response in JSON format + * @param nodeType the type of SEEK node to search for + * @param searchTerm the term to search for + * @return JsonNode of the server's response + */ + private JsonNode genericSearch(String nodeType, String searchTerm) throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/search"; URIBuilder builder = new URIBuilder(endpoint); - builder.setParameter("q", title).setParameter("search_type", nodeType); + builder.setParameter("q", searchTerm).setParameter("search_type", nodeType); HttpRequest request = HttpRequest.newBuilder() .uri(builder.build()) @@ -441,22 +469,26 @@ private String searchNodeWithTitle(String nodeType, String title) .GET().build(); HttpResponse response = HttpClient.newBuilder().build() .send(request, BodyHandlers.ofString()); - System.err.println("searching for: "+title+" ("+nodeType+")"); if(response.statusCode() == 200) { - JsonNode rootNode = new ObjectMapper().readTree(response.body()); - JsonNode hits = rootNode.path("data"); - for (Iterator it = hits.elements(); it.hasNext(); ) { - JsonNode hit = it.next(); - if(hit.get("attributes").get("title").asText().equals(title)) { - return hit.get("id").asText(); - } - } - throw new RuntimeException("Matching "+nodeType+" title was not found : " + title); + return new ObjectMapper().readTree(response.body()); } else { throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } } + private String searchNodeWithTitle(String nodeType, String title) + throws URISyntaxException, IOException, InterruptedException { + JsonNode result = genericSearch(nodeType, title); + JsonNode hits = result.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + if (hit.get("attributes").get("title").asText().equals(title)) { + return hit.get("id").asText(); + } + } + throw new RuntimeException("Matching " + nodeType + " title was not found : " + title); + } + /** * Searches for assays containing a search term and returns a list of found assay ids * @param searchTerm the search term that should be in the assay properties - e.g. an openBIS id @@ -467,9 +499,147 @@ private String searchNodeWithTitle(String nodeType, String title) */ public List searchAssaysContainingKeyword(String searchTerm) throws URISyntaxException, IOException, InterruptedException { - String endpoint = apiURL+"/search/"; + + JsonNode result = genericSearch("assays", "*"+searchTerm+"*"); + + JsonNode hits = result.path("data"); + List assayIDs = new ArrayList<>(); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + assayIDs.add(hit.get("id").asText()); + } + return assayIDs; + } + + /** + * Updates information of an existing assay, its samples and attached assets. Missing samples and + * assets are created, but nothing missing from the new structure is deleted from SEEK. + * @param nodeWithChildren the translated Seek structure as it should be once the update is done + * @param assayID the assay id of the existing assay, that should be compared to the new structure + * @return information necessary to make post registration updates in openBIS and upload missing + * data to newly created assets. In the case of the update use case, only newly created objects + * will be contained in the return object. + */ + public SeekStructurePostRegistrationInformation updateNode(SeekStructure nodeWithChildren, + String assayID) throws URISyntaxException, IOException, InterruptedException { + JsonNode assayData = fetchAssayData(assayID).get("data"); + Map sampleInfos = collectSampleInformation(assayData); + + // compare samples + Map newSamplesWithReferences = nodeWithChildren.getSamplesWithOpenBISReference(); + + List samplesToCreate = new ArrayList<>(); + for (ISASample newSample : newSamplesWithReferences.keySet()) { + String openBisID = newSamplesWithReferences.get(newSample); + SampleInformation existingSample = sampleInfos.get(openBisID); + if (existingSample == null) { + samplesToCreate.add(newSample); + System.out.printf("%s not found in SEEK. It will be created.%n", openBisID); + } else { + Map newAttributes = newSample.fetchCopyOfAttributeMap(); + for (String key : newAttributes.keySet()) { + Object newValue = newAttributes.get(key); + Object oldValue = existingSample.getAttributes().get(key); + + boolean oldEmpty = oldValue == null || oldValue.toString().isEmpty(); + boolean newEmpty = newValue == null || newValue.toString().isEmpty(); + if ((!oldEmpty && !newEmpty) && !newValue.equals(oldValue)) { + System.out.printf("Mismatch found in attributes of %s. Sample will be updated.%n", + openBisID); + newSample.setAssayIDs(List.of(assayID)); + updateSample(newSample, existingSample.getSeekID()); + } + } + } + } + + // compare assets + Map assetInfos = collectAssetInformation(assayData); + Map newAssetsToFiles = nodeWithChildren.getISAFileToDatasetFiles(); + + List assetsToCreate = new ArrayList<>(); + for (GenericSeekAsset newAsset : newAssetsToFiles.keySet()) { + DataSetFile file = newAssetsToFiles.get(newAsset); + String newPermId = file.getDataSetPermId().getPermId(); + if (!assetInfos.containsKey(newPermId)) { + assetsToCreate.add(newAsset); + System.out.printf("Assets with Dataset PermId %s not found in SEEK. File %s from this " + + "Dataset will be created.%n", newPermId, newAsset.getFileName()); + } + } + Map sampleIDsWithEndpoints = new HashMap<>(); + for (ISASample sample : samplesToCreate) { + sample.setAssayIDs(Collections.singletonList(assayID)); + String sampleEndpoint = createSample(sample); + sampleIDsWithEndpoints.put(newSamplesWithReferences.get(sample), sampleEndpoint); + } + List assetsToUpload = new ArrayList<>(); + for (GenericSeekAsset asset : assetsToCreate) { + asset.withAssays(Collections.singletonList(assayID)); + assetsToUpload.add(createAsset(newAssetsToFiles.get(asset).getDataSetPermId().getPermId(), + asset)); + } + Map> datasetIDsWithEndpoints = new HashMap<>(); + + for (AssetToUpload asset : assetsToUpload) { + String endpointWithoutBlob = blobEndpointToAssetURL(asset.getBlobEndpoint()); + String dsCode = asset.getDataSetCode(); + if (datasetIDsWithEndpoints.containsKey(dsCode)) { + datasetIDsWithEndpoints.get(dsCode).add(endpointWithoutBlob); + } else { + datasetIDsWithEndpoints.put(dsCode, new HashSet<>( + List.of(endpointWithoutBlob))); + } + } + + String assayEndpoint = apiURL + "/assays/" + assayID; + AssayWithQueuedAssets assayWithQueuedAssets = + new AssayWithQueuedAssets(assayEndpoint, assetsToUpload); + String expID = nodeWithChildren.getAssayWithOpenBISReference().getRight(); + Pair experimentIDWithEndpoint = new ImmutablePair<>(expID, assayEndpoint); + + return new SeekStructurePostRegistrationInformation(assayWithQueuedAssets, + experimentIDWithEndpoint, sampleIDsWithEndpoints, datasetIDsWithEndpoints); + } + + private Map collectAssetInformation(JsonNode assayData) + throws URISyntaxException, IOException, InterruptedException { + List assetTypes = new ArrayList<>(Arrays.asList("data_files", "models", "sops", + "documents", "publications")); + Map assets = new HashMap<>(); + JsonNode relationships = assayData.get("relationships"); + for(String type : assetTypes) { + for (Iterator it = relationships.get(type).get("data").elements(); it.hasNext(); ) { + String assetID = it.next().get("id").asText(); + AssetInformation assetInfo = fetchAssetInformation(assetID, type); + if(assetInfo.getOpenbisPermId()!=null) { + assets.put(assetInfo.getOpenbisPermId(), assetInfo); + } else { + System.out.printf("No Dataset permID found for existing %s %s (id: %s)%n" + + "This asset will be treated as if it would not exist in the update.%n", + type, assetInfo.getTitle(), assetID); + } + } + } + return assets; + } + + private Map collectSampleInformation(JsonNode assayData) + throws URISyntaxException, IOException, InterruptedException { + Map samples = new HashMap<>(); + JsonNode relationships = assayData.get("relationships"); + for (Iterator it = relationships.get("samples").get("data").elements(); it.hasNext(); ) { + String sampleID = it.next().get("id").asText(); + SampleInformation info = fetchSampleInformation(sampleID); + samples.put(info.getOpenBisIdentifier(), info); + } + return samples; + } + + private AssetInformation fetchAssetInformation(String assetID, String assetType) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/"+assetType+"/"+assetID; URIBuilder builder = new URIBuilder(endpoint); - builder.setParameter("q", searchTerm).setParameter("type", "assays"); HttpRequest request = HttpRequest.newBuilder() .uri(builder.build()) @@ -480,36 +650,113 @@ public List searchAssaysContainingKeyword(String searchTerm) HttpResponse response = HttpClient.newBuilder().build() .send(request, BodyHandlers.ofString()); if(response.statusCode() == 200) { - JsonNode rootNode = new ObjectMapper().readTree(response.body()); - JsonNode hits = rootNode.path("data"); - List assayIDs = new ArrayList<>(); - for (Iterator it = hits.elements(); it.hasNext(); ) { - JsonNode hit = it.next(); - assayIDs.add(hit.get("id").asText()); + JsonNode attributes = new ObjectMapper().readTree(response.body()).get("data").get("attributes"); + String title = attributes.get("title").asText(); + String description = attributes.get("description").asText(); + AssetInformation result = new AssetInformation(assetID, assetType, title, description); + Optional permID = tryParseDatasetPermID(title); + if(permID.isPresent()) { + result.setOpenbisPermId(permID.get()); + } else { + tryParseDatasetPermID(description).ifPresent(result::setOpenbisPermId); } - return assayIDs; + return result; + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + private Optional tryParseDatasetPermID(String input) { + Matcher titleMatcher = OpenbisConnector.datasetCodePattern.matcher(input); + if(titleMatcher.find()) { + return Optional.of(titleMatcher.group()); + } + return Optional.empty(); + } + + private SampleInformation fetchSampleInformation(String sampleID) throws URISyntaxException, + IOException, InterruptedException { + String endpoint = apiURL+"/samples/"+sampleID; + URIBuilder builder = new URIBuilder(endpoint); + + HttpRequest request = HttpRequest.newBuilder() + .uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + JsonNode attributeNode = new ObjectMapper().readTree(response.body()).get("data").get("attributes"); + //title is openbis identifier - this is also added to attribute_map under the name: + //App.configProperties.get("seek_openbis_sample_title"); + String openBisId = attributeNode.get("title").asText(); + Map attributesMap = new ObjectMapper() + .convertValue(attributeNode.get("attribute_map"), Map.class); + return new SampleInformation(sampleID, openBisId, attributesMap); } else { throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } } - public String updateNode(SeekStructure nodeWithChildren, String assayID, boolean transferData) { - //updateAssay(nodeWithChildren.getAssay()); - return apiURL+"/assays/"+assayID; + private JsonNode fetchAssayData(String assayID) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/assays/"+assayID; + URIBuilder builder = new URIBuilder(endpoint); + + HttpRequest request = HttpRequest.newBuilder() + .uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + return new ObjectMapper().readTree(response.body()); + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } } - public AssayWithQueuedAssets createNode(SeekStructure nodeWithChildren, boolean transferData) + public SeekStructurePostRegistrationInformation createNode(SeekStructure nodeWithChildren) throws URISyntaxException, IOException, InterruptedException { - String assayID = addAssay(nodeWithChildren.getAssay()); - for(ISASample sample : nodeWithChildren.getSamples()) { - sample.setAssayIDs(Arrays.asList(assayID)); - createSample(sample); + + Pair assayIDPair = nodeWithChildren.getAssayWithOpenBISReference(); + + String assayID = addAssay(assayIDPair.getKey()); + String assayEndpoint = apiURL+"/assays/"+assayID; + Pair experimentIDWithEndpoint = + new ImmutablePair<>(assayIDPair.getValue(), assayEndpoint); + + Map sampleIDsWithEndpoints = new HashMap<>(); + Map samplesWithReferences = nodeWithChildren.getSamplesWithOpenBISReference(); + for(ISASample sample : samplesWithReferences.keySet()) { + sample.setAssayIDs(Collections.singletonList(assayID)); + String sampleEndpoint = createSample(sample); + sampleIDsWithEndpoints.put(samplesWithReferences.get(sample), sampleEndpoint); } Map isaToFileMap = nodeWithChildren.getISAFileToDatasetFiles(); - return new AssayWithQueuedAssets(apiURL+"/assays/"+assayID, - createAssetsForAssays(isaToFileMap, Arrays.asList(assayID))); + AssayWithQueuedAssets assayWithQueuedAssets = new AssayWithQueuedAssets(assayEndpoint, + createAssetsForAssays(isaToFileMap, Collections.singletonList(assayID))); + + Map> datasetIDsWithEndpoints = new HashMap<>(); + + for(AssetToUpload asset : assayWithQueuedAssets.getAssets()) { + String endpointWithoutBlob = blobEndpointToAssetURL(asset.getBlobEndpoint()); + String dsCode = asset.getDataSetCode(); + if(datasetIDsWithEndpoints.containsKey(dsCode)) { + datasetIDsWithEndpoints.get(dsCode).add(endpointWithoutBlob); + } else { + datasetIDsWithEndpoints.put(dsCode, new HashSet<>( + List.of(endpointWithoutBlob))); + } + } + return new SeekStructurePostRegistrationInformation(assayWithQueuedAssets, + experimentIDWithEndpoint, sampleIDsWithEndpoints, datasetIDsWithEndpoints); } public OpenbisSeekTranslator getTranslator() { @@ -521,11 +768,18 @@ public static class AssetToUpload { private final String blobEndpoint; private final String filePath; private final String openBISDataSetCode; + private final long fileSizeInBytes; - public AssetToUpload(String blobEndpoint, String filePath, String openBISDataSetCode) { + public AssetToUpload(String blobEndpoint, String filePath, String openBISDataSetCode, + long fileSizeInBytes) { this.blobEndpoint = blobEndpoint; this.filePath = filePath; this.openBISDataSetCode = openBISDataSetCode; + this.fileSizeInBytes = fileSizeInBytes; + } + + public long getFileSizeInBytes() { + return fileSizeInBytes; } public String getFilePath() { @@ -540,4 +794,38 @@ public String getDataSetCode() { return openBISDataSetCode; } } + + public class SeekStructurePostRegistrationInformation { + + private final AssayWithQueuedAssets assayWithQueuedAssets; + private final Pair experimentIDWithEndpoint; + private final Map sampleIDsWithEndpoints; + private final Map> datasetIDsWithEndpoints; + + public SeekStructurePostRegistrationInformation(AssayWithQueuedAssets assayWithQueuedAssets, + Pair experimentIDWithEndpoint, Map sampleIDsWithEndpoints, + Map> datasetIDsWithEndpoints) { + this.assayWithQueuedAssets = assayWithQueuedAssets; + this.experimentIDWithEndpoint = experimentIDWithEndpoint; + this.sampleIDsWithEndpoints = sampleIDsWithEndpoints; + this.datasetIDsWithEndpoints = datasetIDsWithEndpoints; + } + + public AssayWithQueuedAssets getAssayWithQueuedAssets() { + return assayWithQueuedAssets; + } + + public Pair getExperimentIDWithEndpoint() { + return experimentIDWithEndpoint; + } + + public Map getSampleIDsWithEndpoints() { + return sampleIDsWithEndpoints; + } + + public Map> getDatasetIDsWithEndpoints() { + return datasetIDsWithEndpoints; + } + + } } diff --git a/src/main/java/life/qbic/model/isa/AbstractISAObject.java b/src/main/java/life/qbic/model/isa/AbstractISAObject.java index 2d68900..c8f952c 100644 --- a/src/main/java/life/qbic/model/isa/AbstractISAObject.java +++ b/src/main/java/life/qbic/model/isa/AbstractISAObject.java @@ -5,6 +5,9 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.module.SimpleModule; +/** + * Used to create the outer "data" node of all SEEK json objects. + */ public abstract class AbstractISAObject { public String toJson(SimpleModule module) throws JsonProcessingException { diff --git a/src/main/java/life/qbic/model/isa/GenericSeekAsset.java b/src/main/java/life/qbic/model/isa/GenericSeekAsset.java index 34b4fef..28bff90 100644 --- a/src/main/java/life/qbic/model/isa/GenericSeekAsset.java +++ b/src/main/java/life/qbic/model/isa/GenericSeekAsset.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * Model class for Seek assets. Contains all mandatory and some optional properties and attributes @@ -22,11 +23,18 @@ public class GenericSeekAsset extends AbstractISAObject { private Attributes attributes; private Relationships relationships; private String assetType; + private long fileSizeInBytes; - public GenericSeekAsset(String assetType, String title, String fileName, List projectIds) { + public GenericSeekAsset(String assetType, String title, String fileName, List projectIds, + long fileSizeInBytes) { this.assetType = assetType; this.attributes = new Attributes(title, fileName); this.relationships = new Relationships(projectIds); + this.fileSizeInBytes = fileSizeInBytes; + } + + public long fileSizeInBytes() { + return fileSizeInBytes; } public GenericSeekAsset withOtherCreators(String otherCreators) { @@ -66,6 +74,13 @@ public String getFileName() { return attributes.getContent_blobs().get(0).getOriginal_filename(); } + public void setDatasetLink(String dataSetLink, boolean transferDate) { + attributes.description = "This asset was imported from openBIS: "+dataSetLink; + if(!transferDate) { + attributes.setExternalLinkToData(dataSetLink); + } + } + private class Relationships { private List projects; @@ -125,14 +140,25 @@ private void generateListJSON(JsonGenerator generator, String name, List private class Attributes { private String title; + private String description; private List contentBlobs = new ArrayList<>(); private String otherCreators = ""; private List dataFormatAnnotations = new ArrayList<>(); - public Attributes(String title, String fileName) { this.title = title; - this.contentBlobs.add(new ContentBlob(fileName)); + ContentBlob blob = new ContentBlob(fileName); + this.contentBlobs.add(blob); + } + + public void setExternalLinkToData(String dataSetLink) { + for(ContentBlob blob : contentBlobs) { + blob.setURL(dataSetLink); + } + } + + public String getDescription() { + return description; } public String getTitle() { @@ -178,6 +204,7 @@ private class ContentBlob { private String originalFilename; private String contentType; + private String url; public ContentBlob(String fileName) { this.originalFilename = fileName; @@ -193,7 +220,36 @@ public String getContent_type() { public String getOriginal_filename() { return originalFilename; } + + public void setURL(String dataSetLink) { + this.url = dataSetLink; + } + + public String getUrl() { + return url; + } + } + } + + @Override + public final boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof GenericSeekAsset)) { + return false; } + + GenericSeekAsset that = (GenericSeekAsset) o; + return Objects.equals(attributes, that.attributes) && Objects.equals( + relationships, that.relationships) && Objects.equals(assetType, that.assetType); } + @Override + public int hashCode() { + int result = Objects.hashCode(attributes); + result = 31 * result + Objects.hashCode(relationships); + result = 31 * result + Objects.hashCode(assetType); + return result; + } } diff --git a/src/main/java/life/qbic/model/isa/ISAAssay.java b/src/main/java/life/qbic/model/isa/ISAAssay.java index ed2a7c0..0de9e48 100644 --- a/src/main/java/life/qbic/model/isa/ISAAssay.java +++ b/src/main/java/life/qbic/model/isa/ISAAssay.java @@ -11,6 +11,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; /** * Model class for ISA Assays. Contains all mandatory and some optional properties and attributes @@ -25,8 +26,10 @@ public class ISAAssay extends AbstractISAObject { private Attributes attributes; private Relationships relationships; + private String title; public ISAAssay(String title, String studyId, String assayClass, URI assayType) { + this.title = title; this.attributes = new Attributes(title, assayClass, assayType); this.relationships = new Relationships(studyId); } @@ -175,7 +178,7 @@ private void generateListJSON(JsonGenerator generator, String name, List tags = new ArrayList<>(); public String description = ""; @@ -245,4 +248,27 @@ public String getUri() { } } + @Override + public final boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ISAAssay)) { + return false; + } + + ISAAssay isaAssay = (ISAAssay) o; + return Objects.equals(attributes, + isaAssay.attributes) && Objects.equals(relationships, isaAssay.relationships) + && Objects.equals(title, isaAssay.title); + } + + @Override + public int hashCode() { + int result = ISA_TYPE.hashCode(); + result = 31 * result + Objects.hashCode(attributes); + result = 31 * result + Objects.hashCode(relationships); + result = 31 * result + Objects.hashCode(title); + return result; + } } diff --git a/src/main/java/life/qbic/model/isa/ISASample.java b/src/main/java/life/qbic/model/isa/ISASample.java index 1663b99..b16e04f 100644 --- a/src/main/java/life/qbic/model/isa/ISASample.java +++ b/src/main/java/life/qbic/model/isa/ISASample.java @@ -10,6 +10,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; /** * Model class for ISA Samples. Contains all mandatory and some optional properties and attributes @@ -23,6 +24,7 @@ public class ISASample extends AbstractISAObject { private Attributes attributes; private Relationships relationships; private final String ISA_TYPE = "samples"; + private String id; public ISASample(String title, Map attributeMap, String sampleTypeId, List projectIds) { @@ -35,6 +37,10 @@ public ISASample withOtherCreators(String otherCreators) { return this; } + public void setSampleID(String seekID) { + this.id = seekID; + } + public void setCreatorIDs(List creatorIDs) { this.relationships.setCreatorIDs(creatorIDs); } @@ -43,6 +49,10 @@ public void setAssayIDs(List assayIDs) { this.relationships.setAssayIDs(assayIDs); } + public String getId() { + return id; + } + public String toJson() throws JsonProcessingException { SimpleModule module = new SimpleModule(); module.addSerializer(Relationships.class, new RelationshipsSerializer()); @@ -61,6 +71,10 @@ public Attributes getAttributes() { return attributes; } + public Map fetchCopyOfAttributeMap() { + return new HashMap<>(attributes.getAttribute_map()); + } + private class Relationships { private String sampleTypeId; @@ -165,4 +179,25 @@ public String getOther_creators() { } } + @Override + public final boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ISASample)) { + return false; + } + + ISASample isaSample = (ISASample) o; + return Objects.equals(attributes, isaSample.attributes) && Objects.equals( + relationships, isaSample.relationships); + } + + @Override + public int hashCode() { + int result = Objects.hashCode(attributes); + result = 31 * result + Objects.hashCode(relationships); + result = 31 * result + ISA_TYPE.hashCode(); + return result; + } } diff --git a/src/main/java/life/qbic/model/isa/ISASampleType.java b/src/main/java/life/qbic/model/isa/ISASampleType.java index 201a263..3ccee05 100644 --- a/src/main/java/life/qbic/model/isa/ISASampleType.java +++ b/src/main/java/life/qbic/model/isa/ISASampleType.java @@ -216,5 +216,15 @@ public String getId() { public String getTitle() { return title; } + + @Override + public String toString() { + return "SampleAttributeType{" + + "id='" + id + '\'' + + ", title='" + title + '\'' + + ", baseType='" + baseType + '\'' + + '}'; + } } + } diff --git a/src/main/java/life/qbic/model/isa/SeekStructure.java b/src/main/java/life/qbic/model/isa/SeekStructure.java index 9dd45b4..29caeef 100644 --- a/src/main/java/life/qbic/model/isa/SeekStructure.java +++ b/src/main/java/life/qbic/model/isa/SeekStructure.java @@ -1,28 +1,44 @@ package life.qbic.model.isa; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; - +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Stores newly created ISA objects for SEEK, as well as their respective openBIS reference. It is + * assumed that these references are Sample and Experiment Identifiers. PermIds of datasets are taken + * from stored DataSetFiles + */ public class SeekStructure { - private ISAAssay assay; - private List samples; - private Map isaToOpenBISFile; + private final Pair assayAndOpenBISReference; + private final Map samplesWithOpenBISReference; + private final Map isaToOpenBISFile; + + public SeekStructure(ISAAssay assay, String openBISReference) { + this.assayAndOpenBISReference = new ImmutablePair<>(assay, openBISReference); + this.samplesWithOpenBISReference = new HashMap<>(); + this.isaToOpenBISFile = new HashMap<>(); + } + + public void addSample(ISASample sample, String openBISReference) { + samplesWithOpenBISReference.put(sample, openBISReference); + } - public SeekStructure(ISAAssay assay, List samples, - Map isaToOpenBISFile) { - this.assay = assay; - this.samples = samples; - this.isaToOpenBISFile = isaToOpenBISFile; + public void addAsset(GenericSeekAsset asset, DataSetFile file) { + isaToOpenBISFile.put(asset, file); } - public ISAAssay getAssay() { - return assay; + public Pair getAssayWithOpenBISReference() { + return assayAndOpenBISReference; } - public List getSamples() { - return samples; + public Map getSamplesWithOpenBISReference() { + return samplesWithOpenBISReference; } public Map getISAFileToDatasetFiles() {