From 4d28df98854fd7108942e52f421d26593139739a Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Thu, 26 Sep 2024 17:30:29 +0200 Subject: [PATCH] wip --- .../io/commandline/AuthenticationOptions.java | 85 +++- .../io/commandline/CommandLineOptions.java | 10 +- .../TransferDataToSeekCommand.java | 244 ++++++++++- .../io/commandline/UploadDatasetCommand.java | 20 +- .../OpenbisExperimentWithDescendants.java | 40 ++ .../qbic/model/OpenbisSeekTranslator.java | 121 ++++++ .../java/life/qbic/model/SeekStructure.java | 34 ++ .../qbic/model/download/OpenbisConnector.java | 71 +++- .../qbic/model/download/SEEKConnector.java | 397 ++++++++++++++++++ .../qbic/model/isa/AbstractISAObject.java | 22 + .../java/life/qbic/model/isa/ISAAssay.java | 257 ++++++++++++ .../java/life/qbic/model/isa/ISADataFile.java | 203 +++++++++ .../java/life/qbic/model/isa/ISASample.java | 168 ++++++++ .../java/life/qbic/model/isa/ISAStudy.java | 158 +++++++ 14 files changed, 1785 insertions(+), 45 deletions(-) create mode 100644 src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java create mode 100644 src/main/java/life/qbic/model/OpenbisSeekTranslator.java create mode 100644 src/main/java/life/qbic/model/SeekStructure.java create mode 100644 src/main/java/life/qbic/model/download/SEEKConnector.java create mode 100644 src/main/java/life/qbic/model/isa/AbstractISAObject.java create mode 100644 src/main/java/life/qbic/model/isa/ISAAssay.java create mode 100644 src/main/java/life/qbic/model/isa/ISADataFile.java create mode 100644 src/main/java/life/qbic/model/isa/ISASample.java create mode 100644 src/main/java/life/qbic/model/isa/ISAStudy.java diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java index 6809a06..1b9c5d0 100644 --- a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java +++ b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java @@ -23,7 +23,9 @@ public class AuthenticationOptions { description = "openBIS user name") private String openbisUser; @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. - PasswordOptions openbisPasswordOptions; + OpenbisPasswordOptions openbisPasswordOptions; + @ArgGroup(multiplicity = "1") + SeekPasswordOptions seekPasswordOptions; @Option( names = {"-as", "-as_url"}, @@ -32,17 +34,21 @@ public class AuthenticationOptions { private String as_url; @Option( - names = {"-dss", "-dss_url"}, + names = {"-dss", "--dss_url"}, description = "DatastoreServer URL", scope = CommandLine.ScopeType.INHERIT) private String dss_url; @Option( - names = {"-config", "-config_file"}, - description = "Config file path to provide openbis server information.", + names = {"-config", "--config_file"}, + description = "Config file path to provide server and user information.", scope = CommandLine.ScopeType.INHERIT) public String configPath; - + @Option( + names = {"-su", "--seek-user"}, + description = "Seek user name (email)", + scope = CommandLine.ScopeType.INHERIT) + private String seekUser; @Option( names = {"-seek-server", "-seek_url"}, description = "SEEK API URL", @@ -52,13 +58,30 @@ public class AuthenticationOptions { public String getOpenbisUser() { if(openbisUser == null & configPath!=null && !configPath.isBlank()) { openbisUser = ReadProperties.getProperties(configPath).get("user"); + } else { + log.error("No openBIS user provided."); + System.exit(2); } return openbisUser; } + public String getSeekUser() { + if(seekUser == null & configPath!=null && !configPath.isBlank()) { + seekUser = ReadProperties.getProperties(configPath).get("seek_user"); + } else { + log.error("No SEEK user/email provided."); + System.exit(2); + } + return seekUser; + } + public String getSeekURL() { - log.error("No URL to the SEEK address provided."); - System.exit(2); + if(seek_url == null & configPath!=null && !configPath.isBlank()) { + seek_url = ReadProperties.getProperties(configPath).get("seek_url"); + } else { + log.error("No URL to the SEEK address provided."); + System.exit(2); + } return seek_url; } @@ -76,6 +99,10 @@ public String getAS() { return as_url; } + public char[] getSeekPassword() { + return seekPasswordOptions.getPassword(); + } + public char[] getOpenbisPassword() { return openbisPasswordOptions.getPassword(); } @@ -83,14 +110,14 @@ public char[] getOpenbisPassword() { /** * official picocli documentation example */ - static class PasswordOptions { - @Option(names = "--password:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") + static class OpenbisPasswordOptions { + @Option(names = "--openbis-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") protected String passwordEnvironmentVariable = ""; - @Option(names = "--password:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") + @Option(names = "--openbis-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") protected String passwordProperty = ""; - @Option(names = "--password", arity = "0", description = "please provide your password", interactive = true) + @Option(names = "--openbis-pw", arity = "0", description = "please provide your openBIS password", interactive = true) protected char[] password = null; /** @@ -116,9 +143,43 @@ char[] getPassword() { System.exit(2); return null; // not reachable due to System.exit in previous line } - } + static class SeekPasswordOptions { + @Option(names = "--seek-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") + protected String passwordEnvironmentVariable = ""; + + @Option(names = "--seek-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") + protected String passwordProperty = ""; + + @Option(names = "--seek-pw", arity = "0", description = "please provide your SEEK password", interactive = true) + protected char[] password = null; + + /** + * Gets the password. If no password is provided, the program exits. + * @return the password provided by the user. + */ + char[] getPassword() { + if (nonNull(password)) { + return password; + } + // System.getProperty(String key) does not work for empty or blank keys. + if (!passwordProperty.isBlank()) { + String systemProperty = System.getProperty(passwordProperty); + if (nonNull(systemProperty)) { + return systemProperty.toCharArray(); + } + } + String environmentVariable = System.getenv(passwordEnvironmentVariable); + if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { + return environmentVariable.toCharArray(); + } + log.error("No password provided. Please provide your password."); + System.exit(2); + return null; // not reachable due to System.exit in previous line + } + + } @Override public String toString() { return new StringJoiner(", ", AuthenticationOptions.class.getSimpleName() + "[", "]") diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index d050ebf..a0dcbad 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -8,11 +8,13 @@ // main command with format specifiers for the usage help message @Command(name = "openbis-scripts", - subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, DownloadPetabCommand.class, - UploadPetabResultCommand.class, UploadDatasetCommand.class, SpaceStatisticsCommand.class }, - description = "A client software for querying openBIS.", - mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) + subcommands = {SampleHierarchyCommand.class, FindDatasetsCommand.class, + DownloadPetabCommand.class, UploadPetabResultCommand.class, UploadDatasetCommand.class, + SpaceStatisticsCommand.class, TransferDataToSeekCommand.class}, + description = "A client software for querying openBIS.", + mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) public class CommandLineOptions { + private static final Logger LOG = LogManager.getLogger(CommandLineOptions.class); @Option(names = {"-V", "--version"}, diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index 5704b3d..ec9bdaf 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -2,35 +2,164 @@ import ch.ethz.sis.openbis.generic.OpenBIS; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.io.File; -import java.util.Collections; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; import life.qbic.App; -import life.qbic.model.DatasetWithProperties; +import life.qbic.model.OpenbisExperimentWithDescendants; +import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.SeekStructure; import life.qbic.model.download.OpenbisConnector; +import life.qbic.model.download.SEEKConnector; +import life.qbic.model.download.SEEKConnector.AssetToUpload; +import org.apache.commons.codec.binary.Base64; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @Command(name = "openbis-to-seek", - description = "Transfers data or metadata from openBIS to SEEK.") + description = + "Transfers metadata and (optionally) data from openBIS to SEEK. Experiments, samples and " + + "dataset information are always transferred together (as assays, samples and one of " + + "several data types in SEEK). Dataset info always links back to the openBIS path of " + + "the respective dataset. The data itself will can transferred and stored in SEEK " + + "using the '-d' flag." + + "To completely exclude some dataset information from being transferred, a " + + "file ('--blacklist') containing dataset codes (from openBIS) can be specified." + + "Unless otherwise specified ('--no-update' flag), the command will try to update " + + "existing nodes in SEEK (recognized by openBIS identifiers in their metadata).") public class TransferDataToSeekCommand implements Runnable { - @Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset (or its metadata) to transfer. Can be found via list-data.") - private String datasetCode; - @Parameters(arity = "1", paramLabel = "seek node", description = "The node in SEEK to which to transfer the dataset.") + @Parameters(arity = "1", paramLabel = "openbis id", description = "The identifier of the " + + "experiment, sample or dataset to transfer.") + private String objectID; + @Option(names = "--blacklist", description = "Path to file specifying by dataset " + + "dataset code which openBIS datasets not to transfer to SEEK. The file must contain one code " + + "per line.") + private String blacklistFile; + @Option(names = "--no-update", description = "Use to specify that existing " + + "information in SEEK for the specified openBIS input should not be updated, but new nodes " + + "created.") + private boolean noUpdate; + /*@Option(names = {"-sn", "--seek-node"}, paramLabel = "seek node", description = + "The target node in SEEK to transfer to. Must correspond to " + + "the type of oopenBIS identifier chosen: experiment - assay; sample - sample; dataset - any of the data types. If no node is specified, " + + "a new data structure will be created in SEEK, starting from the related experiment.") private String seekNode; - @Option(names = { "-d", "--data"}, usageHelp = true, description = "Transfers the data itself to SEEK along with the metadata") + */ + @Option(names = {"-d", "--data"}, description = + "Transfers the data itself to SEEK along with the metadata. " + + "Otherwise only the link(s) to the openBIS object will be created in SEEK.") private boolean transferData; @Mixin AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisConnector openbis; + SEEKConnector seek; + OpenbisSeekTranslator translator = new OpenbisSeekTranslator(); - @Override - public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); - OpenbisConnector openbis = new OpenbisConnector(authentication); + @Override + public void run() { + System.out.println("auth..."); + + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getAS(), auth.getDSS()); + System.out.println("openbis..."); + + openbis = new OpenbisConnector(authentication); + + boolean isSample = false; + boolean isDataSet = false; + System.out.println("search for experiment..."); + + boolean isExperiment = experimentExists(objectID); + if (!isExperiment && sampleExists(objectID)) { + isSample = true; + } + + if (!isExperiment && !isSample && datasetsExist(Arrays.asList(objectID))) { + isDataSet = true; + } + + if (!isSample && !isExperiment && !isDataSet) { + System.out.printf( + "%s could not be found in openBIS. Make sure you either specify an experiment, sample or dataset%n", + objectID); + return; + } + System.out.println("Searching done..."); +/* + if (isExperiment) { + if (seekNode != null && !seekNode.contains("assays")) { + System.out.printf( + "Seek node %s does not correspond to the provided openBIS experiment identifier. " + + "Please select an assay node.%n", objectID); + return; + } + } + if (isSample) { + if (seekNode != null && !seekNode.contains("samples")) { + System.out.printf( + "Seek node %s does not correspond to the provided openBIS sample identifier. " + + "Please select a sample node.%n", objectID); + return; + } + } + + */ + + byte[] httpCredentials = Base64.encodeBase64( + (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); + try { + seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "Default Project", + "lisym default study"); + } catch (URISyntaxException | IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + /* + if (seekNode != null) { + try { + if (!seek.endPointExists(seekNode)) { + System.out.println(seekNode + " could not be found"); + return; + } else { + if (isExperiment) { + Experiment experimentWithSamplesAndDatasets = openbis.getExperimentWithDescendants( + objectID); + //SeekStructure seekStructure = translator.translate(seekNode, experimentWithSamplesAndDatasets); + //seek.fillAssayWithSamplesAndDatasets(seekNode, seekStructure); + } + } + } catch (URISyntaxException | InterruptedException | IOException e) { + throw new RuntimeException(e); + } + } + + */ + try { + OpenbisExperimentWithDescendants experiment = openbis.getExperimentWithDescendants(objectID); + String assayID = getAssayIDForOpenBISExperiment(experiment.getExperiment()); + List blacklist = parseBlackList(blacklistFile); + if(assayID.isBlank()) { + createNewNodes(experiment, blacklist); + } else { + updateNodes(experiment, assayID, blacklist); + } + } catch (URISyntaxException | InterruptedException | IOException e) { + throw new RuntimeException(e); + } + + /* List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); @@ -38,6 +167,7 @@ public void run() { System.out.println(datasetCode+" not found"); return; } + DatasetWithProperties result = new DatasetWithProperties(datasets.get(0)); Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", result.getExperiment().getIdentifier()); @@ -53,9 +183,97 @@ public void run() { cleanupTemp(new File(tmpPath)); +*/ + System.out.println("Done"); + } - System.out.println("Done"); + private List parseBlackList(String blacklistFile) { + List result = new ArrayList<>(); + if(blacklistFile == null) { + return result; + } + // trim whitespace, skip empty lines + try (Stream lines = Files.lines(Paths.get(blacklistFile)) + .map(String::trim) + .filter(s -> !s.isBlank())) { + return lines.collect(Collectors.toList()); + } catch (IOException e) { + throw new RuntimeException(blacklistFile+" could not be found or read."); } + } + + private void updateNodes(OpenbisExperimentWithDescendants experiment, String assayID, List blacklist) { + System.err.println("updating nodes of assay "+assayID); + SeekStructure nodeWithChildren = translator.translate(experiment, blacklist); + String updatedEndpoint = seek.updateNode(nodeWithChildren, assayID, transferData); + System.out.printf("%s was successfully updated.%n", endpoint); + } + + private void createNewNodes(OpenbisExperimentWithDescendants experiment, List blacklist) + throws URISyntaxException, IOException, InterruptedException { + System.err.println("creating new nodes"); + SeekStructure nodeWithChildren = translator.translate(experiment, blacklist); + List assetsToUpload = seek.createNode(nodeWithChildren, transferData); + if(transferData) { + for(AssetToUpload asset : assetsToUpload) { + System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); + String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), + () -> openbis.streamDataset(asset.getDataSetCode(), asset.getFilePath())); + System.out.printf("File stored here: %s%n", fileURL); + } + } + System.out.printf("%s was successfully created.%n", endpoint); + } + + private boolean sampleExists(String objectID) { + return openbis.sampleExists(objectID); + } + + private boolean datasetsExist(List datasetCodes) { + return openbis.findDataSets(datasetCodes).size() == datasetCodes.size(); + } + + private boolean experimentExists(String experimentID) { + return openbis.experimentExists(experimentID); + } + + private String getAssayIDForOpenBISExperiment(Experiment experiment) + throws URISyntaxException, IOException, InterruptedException { + // the perm id is unique and afaik not used by scientists. it is highly unlikely that it would + // "accidentally" be part of another title or description. however, care should be taken here, + // because if a perm id is found in the wrong SEEK node, meta-information in SEEK could be + // overwritten or samples/data added to the wrong assay. + String permID = experiment.getPermId().getPermId(); + List assayIDs = seek.searchAssaysContainingKeyword(permID); + if(assayIDs.isEmpty()) { + return ""; + } + if(assayIDs.size() == 1) { + return assayIDs.get(0); + } + throw new RuntimeException("Search term "+permID+ " was found in more than one assay: "+assayIDs); + } + + private void sendDatasetToSeek(String datasetCode, String assayID) + throws URISyntaxException, IOException, InterruptedException { + assayID = "3"; + System.out.println("Searching dataset in openBIS..."); + List datasets = openbis.findDataSets( + Arrays.asList(datasetCode)); + if(datasets.isEmpty()) { + return; + } + DataSet dataset = datasets.get(0); + List files = openbis.getDatasetFiles(dataset); + List assets = seek.createAssets(files, dataset.getType().getCode(), + Arrays.asList(assayID)); + for(AssetToUpload asset : assets) { + System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); + String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), + () -> openbis.streamDataset(datasetCode, asset.getFilePath())); + System.out.printf("File stored here: %s%n", fileURL); + } + } private void cleanupTemp(File tmpFolder) { File[] files = tmpFolder.listFiles(); diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index f165c9a..9e2d3a9 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -39,21 +39,13 @@ public void run() { System.out.printf("Path %s could not be found%n", dataPath); return; } - boolean attachToSample = OpenbisConnector.sampleIdPattern.matcher(objectID).find(); - boolean attachToExperiment = false; - if(!attachToSample) { - attachToExperiment = OpenbisConnector.experimentIdPattern.matcher(objectID).find(); + boolean attachToSample = false; + boolean attachToExperiment = experimentExists(objectID); + if(sampleExists(objectID)) { + attachToSample = true; } - if(!attachToExperiment && !attachToSample) { - System.out.printf("%s is neither a valid experiment nor sample identifier%n", objectID); - return; - } - if(attachToExperiment && !experimentExists(objectID)) { - System.out.printf("Experiment with identifier %s could not be found%n", objectID); - return; - } - if(attachToSample && !sampleExists(objectID)) { - System.out.printf("Sample object with identifier %s could not be found%n", objectID); + if(!attachToSample && !attachToExperiment) { + System.out.printf("%s could not be found in openBIS.%n", objectID); return; } if(!datasetsExist(parents)) { diff --git a/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java new file mode 100644 index 0000000..86399ec --- /dev/null +++ b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java @@ -0,0 +1,40 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.util.List; +import java.util.Map; + +public class OpenbisExperimentWithDescendants { + + private Experiment experiment; + private List samples; + private List datasets; + private Map> datasetCodeToFiles; + + public OpenbisExperimentWithDescendants(Experiment experiment, List samples, + List datasets, Map> datasetCodeToFiles) { + this.experiment = experiment; + this.samples = samples; + this.datasets = datasets; + this.datasetCodeToFiles = datasetCodeToFiles; + } + + public Experiment getExperiment() { + return experiment; + } + + public List getSamples() { + return samples; + } + + public List getDatasets() { + return datasets; + } + + public List getFilesForDataset(String permID) { + return datasetCodeToFiles.get(permID); + } +} diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java new file mode 100644 index 0000000..a02317d --- /dev/null +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -0,0 +1,121 @@ +package life.qbic.model; + +import static java.util.Map.entry; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import life.qbic.model.isa.ISADataFile; +import life.qbic.model.isa.ISASample; + +public class OpenbisSeekTranslator { + + private final String DEFAULT_PROJECT_ID; + private final String DEFAULT_STUDY_ID; + + public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { + this.DEFAULT_PROJECT_ID = defaultProjectID; + this.DEFAULT_STUDY_ID = defaultStudyID; + } + + Map experimentTypeToAssayClass = Map.ofEntries( + entry("00_MOUSE_DATABASE", "EXP"), + entry("00_PATIENT_DATABASE", "EXP"), + entry("00_STANDARD_OPERATING_PROTOCOLS", "EXP"), + entry("01_BIOLOGICAL_EXPERIMENT", "EXP"), + entry("02_MASSSPECTROMETRY_EXPERIMENT", "EXP"),//where is petab modeling attached? + entry("03_HISTOLOGICAL_ANALYSIS", "EXP"), + entry("04_MICRO_CT", "EXP") + ); + + Map experimentTypeToAssayType = Map.ofEntries( + entry("00_MOUSE_DATABASE", ""), + entry("00_PATIENT_DATABASE", ""),//if this is related to measured data, attach it as sample to the assay + entry("00_STANDARD_OPERATING_PROTOCOLS", ""), + entry("01_BIOLOGICAL_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Cultivation_experiment"), + entry("02_MASSSPECTROMETRY_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Proteomics"), + entry("03_HISTOLOGICAL_ANALYSIS", ""), + entry("04_MICRO_CT", "") + ); + + Map fileExtensionToDataFormat = Map.ofEntries( + entry("fastq.gz", "http://edamontology.org/format_1930"), + entry("fastq", "http://edamontology.org/format_1930"), + entry("json", "http://edamontology.org/format_3464"), + entry("yaml", "http://edamontology.org/format_3750"), + entry("raw", "http://edamontology.org/format_3712"), + entry("tsv", "http://edamontology.org/format_3475"), + entry("csv", "http://edamontology.org/format_3752") + ); + + Map datasetTypeToAssetType = Map.ofEntries( + entry("ANALYSIS_NOTEBOOK", "Document"), + entry("ANALYZED_DATA", "Data_file"), + entry("ATTACHMENT", "Document"), + entry("ELN_PREVIEW", ""), + entry("EXPERIMENT_PROTOCOL", "SOP"), + entry("EXPERIMENT_RESULT", "Document"), + entry("HISTOLOGICAL_SLIDE", "Data_file"), + entry("IB_DATA", "Data_file"), + entry("LUMINEX_DATA", "Data_file"), + entry("MS_DATA_ANALYZED", "Data_file"), + entry("MS_DATA_RAW", "Data_file"), + entry("OTHER_DATA", "Document"), + entry("PROCESSED_DATA", "Data_file"), + entry("PUBLICATION_DATA", "Publication"), + entry("QPCR_DATA", "Data_file"), + entry("RAW_DATA", "Data_file"), + entry("SOURCE_CODE", "Document"), + entry("TEST_CONT", ""), + entry("TEST_DAT", ""), + entry("UNKNOWN", "") + ); + + public SeekStructure translate(String seekNode, Experiment experimentWithSamplesAndDatasets) { + Experiment exp = experimentWithSamplesAndDatasets; + exp.getType(); + return null; + //new ISAAssay(exp.getCode(), ) + } + + public ISADataFile translate(String seekNode, DataSet dataset) { + return null; //new ISADataFile(); + } + + public String assetForDatasetType(String datasetType) { + return "Data_file";//TODO + } + + public String dataFormatAnnotationForExtension(String fileExtension) { + return fileExtensionToDataFormat.get(fileExtension); + } + + public SeekStructure translate(Experiment experiment, List blacklist) { + System.err.println(experiment.getCode()); + + for(Sample sample : experiment.getSamples()) { + String sampleType = getSampleType(sample.getType()); + Map attributes = new HashMap<>(); + ISASample isaSample = new ISASample(attributes, sampleType, + Collections.singletonList(DEFAULT_PROJECT_ID)); + System.err.println(sample.getCode()); + } + for(DataSet dataset : experiment.getDataSets()) { + System.err.println(dataset.getCode()); + } + System.err.println("blacklisted:"); + for(String dataset : blacklist) { + System.err.println(dataset); + } + return null;//TODO + } + + private String getSampleType(SampleType type) { + } +} diff --git a/src/main/java/life/qbic/model/SeekStructure.java b/src/main/java/life/qbic/model/SeekStructure.java new file mode 100644 index 0000000..5d7cc46 --- /dev/null +++ b/src/main/java/life/qbic/model/SeekStructure.java @@ -0,0 +1,34 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.util.List; +import java.util.Map; +import life.qbic.model.isa.ISAAssay; +import life.qbic.model.isa.ISADataFile; +import life.qbic.model.isa.ISASample; + +public class SeekStructure { + + private ISAAssay assay; + private List samples; + private List dataFiles; + private Map isaToOpenBISFile; + private Map isaFileToAssetType; + //ISASample isaSample = new ISASample(map, sampleTypeID, Arrays.asList(projectID)); + + public ISAAssay getAssay() { + return assay; + } + + public List getSamples() { + return samples; + } + + public Map getIsaToOpenBISFiles() { + return isaToOpenBISFile; + } + + public String getAssetType(ISADataFile dataFile) { + return isaFileToAssetType.get(dataFile); + } +} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index c73d796..e50dd12 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -25,8 +25,10 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownload; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadOptions; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadReader; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.DataSetFilePermId; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.IDataSetFileId; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -40,6 +42,8 @@ import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; +import life.qbic.model.DatasetWithProperties; +import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.SampleTypeConnection; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -49,8 +53,9 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); private final OpenBIS openBIS; - public final static Pattern experimentIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); - public final static Pattern sampleIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); + //public final static Pattern experimentIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); + //public final static Pattern sampleIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); + public static Pattern datasetCodePattern = Pattern.compile("[0-9]^{17}-[0-9]+"); public OpenbisConnector(OpenBIS authentication) { this.openBIS = authentication; @@ -161,6 +166,22 @@ public File downloadDataset(String targetPath, String datasetID) { return new File(targetPath); } + public InputStream streamDataset(String datasetCode, String filePath) { + DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); + IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetCode), + filePath); + + // Setting recursive flag to true will return both subfolders and files + options.setRecursive(true); + + // Read the contents and print them out + InputStream stream = openBIS.downloadFiles(new ArrayList<>(List.of(fileToDownload)), + options); + + DataSetFileDownloadReader reader = new DataSetFileDownloadReader(stream); + return reader.read().getInputStream(); + } + public Map queryFullSampleHierarchy(List spaces) { Map hierarchy = new HashMap<>(); if (spaces.isEmpty()) { @@ -370,9 +391,14 @@ public List findDataSets(List codes) { criteria.withCodes().thatIn(codes); DataSetFetchOptions options = new DataSetFetchOptions(); options.withExperiment(); + options.withType(); return openBIS.searchDataSets(criteria, options).getObjects(); } + public boolean datasetExists(String code) { + return !findDataSets(new ArrayList<>(Arrays.asList(code))).isEmpty(); + } + public boolean experimentExists(String experimentID) { ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); criteria.withIdentifier().thatEquals(experimentID); @@ -388,4 +414,45 @@ public boolean sampleExists(String objectID) { return !openBIS.searchSamples(criteria, new SampleFetchOptions()).getObjects() .isEmpty(); } + + public OpenbisExperimentWithDescendants getExperimentWithDescendants(String experimentID) { + ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); + criteria.withIdentifier().thatEquals(experimentID); + + ExperimentFetchOptions fetchOptions = new ExperimentFetchOptions(); + fetchOptions.withType(); + fetchOptions.withProject(); + fetchOptions.withProperties(); + DataSetFetchOptions dataSetFetchOptions = new DataSetFetchOptions(); + dataSetFetchOptions.withType(); + dataSetFetchOptions.withRegistrator(); + SampleFetchOptions sampleFetchOptions = new SampleFetchOptions(); + sampleFetchOptions.withProperties(); + sampleFetchOptions.withDataSetsUsing(dataSetFetchOptions); + fetchOptions.withDataSetsUsing(dataSetFetchOptions); + fetchOptions.withSamplesUsing(sampleFetchOptions); + + Experiment experiment = openBIS.searchExperiments(criteria, fetchOptions).getObjects().get(0); + + Map> datasetCodeToFiles = new HashMap<>(); + for(DataSet dataset : experiment.getDataSets()) { + datasetCodeToFiles.put(dataset.getPermId(), getDatasetFiles(dataset)); + } + + return new OpenbisExperimentWithDescendants(experiment, experiment.getSamples(), + experiment.getDataSets() + .stream().map(DatasetWithProperties::new) + .collect(Collectors.toList()), datasetCodeToFiles); + } + + public List getDatasetFiles(DataSet dataset) { + DataSetFileSearchCriteria criteria = new DataSetFileSearchCriteria(); + + DataSetSearchCriteria dataSetCriteria = criteria.withDataSet().withOrOperator(); + dataSetCriteria.withCode().thatEquals(dataset.getCode()); + + SearchResult result = openBIS.searchFiles(criteria, new DataSetFileFetchOptions()); + + return result.getObjects(); + } } diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java new file mode 100644 index 0000000..298364e --- /dev/null +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -0,0 +1,397 @@ +package life.qbic.model.download; + +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; +import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.SeekStructure; +import life.qbic.model.isa.ISAAssay; +import life.qbic.model.isa.ISADataFile; +import life.qbic.model.isa.ISASample; +import life.qbic.model.isa.ISAStudy; +import org.apache.http.client.utils.URIBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class SEEKConnector { + + private static final Logger LOG = LogManager.getLogger(SEEKConnector.class); + private String apiURL; + private byte[] credentials; + private OpenbisSeekTranslator translator; + + public SEEKConnector(String apiURL, byte[] httpCredentials, String defaultProjectTitle, + String defaultStudyTitle) throws URISyntaxException, IOException, InterruptedException { + this.apiURL = apiURL; + this.credentials = httpCredentials; + translator = new OpenbisSeekTranslator(searchNodeWithTitle("projects", defaultProjectTitle), + searchNodeWithTitle("studies", defaultStudyTitle)); + } + + public String addAssay(ISAAssay assay) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/assays"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, assay.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + + public String createStudy(ISAStudy study) + throws IOException, URISyntaxException, InterruptedException, IOException { + String endpoint = apiURL+"/studies"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, study.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + + private HttpRequest buildAuthorizedPOSTRequest(String endpoint, String body) throws URISyntaxException { + return HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .POST(HttpRequest.BodyPublishers.ofString(body)).build(); + } + + public boolean studyExists(String id) throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/studies/"+id; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + return response.statusCode() == 200; + } + + /* + -datatype by extension +-assay equals experiment +-investigation: pre-created in SEEK? +-study: project +-patient id should be linked somehow, maybe gender? +-flexible object type to sample type? + */ + + public String createSample(ISASample isaSample) throws URISyntaxException, IOException, + InterruptedException { + String endpoint = apiURL+"/samples"; + HashMap map = new HashMap<>(); + //map.put("title", "really?"); + map.put("test_attribute", "okay then"); + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, isaSample.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=201) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + + private AssetToUpload createDataFileAsset(String datasetCode, ISADataFile data) + throws IOException, URISyntaxException, InterruptedException { + String endpoint = apiURL+"/data_files"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, data.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=201 && response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data") + .path("attributes") + .path("content_blobs") + .path(0).path("link"); + return new AssetToUpload(idNode.asText(), data.getFileName(), datasetCode); + } + + @Deprecated + private void uploadFileContent(String assetType, String assetID, String blobID, String file) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/"+assetType+"/"+assetID+"/content_blobs/"+blobID; + + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/octet-stream") + .headers("Accept", "application/octet-stream") + .headers("Authorization", "Basic " + new String(credentials)) + .PUT(BodyPublishers.ofFile(new File(file).toPath())).build(); + + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + public String uploadStreamContent(String blobEndpoint, + Supplier streamSupplier) + throws URISyntaxException, IOException, InterruptedException { + + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(blobEndpoint)) + .headers("Content-Type", "application/octet-stream") + .headers("Accept", "application/octet-stream") + .headers("Authorization", "Basic " + new String(credentials)) + .PUT(BodyPublishers.ofInputStream(streamSupplier)).build(); + + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } else { + String fileURL = blobEndpoint.split("content_blobs")[0]; + return fileURL; + } + } + + public boolean endPointExists(String endpoint) + throws URISyntaxException, IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + return response.statusCode() == 200; + } + + /** + * Creates an optional asset for a file from an openBIS dataset. Folders are ignored. + * @param file + * @param assetType + * @param assays + * @return + * @throws IOException + * @throws URISyntaxException + * @throws InterruptedException + */ + public Optional createAssetForFile(DataSetFile file, String assetType, + List assays) + throws IOException, URISyntaxException, InterruptedException { + if(!file.getPath().isBlank() && !file.isDirectory()) { + File f = new File(file.getPath()); + String datasetCode = file.getDataSetPermId().toString(); + String assetName = datasetCode+": "+f.getName(); + ISADataFile isaFile = new ISADataFile(assetName, file.getPath(), + Arrays.asList("1")); + isaFile.withAssays(assays); + String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); + String annotation = translator.dataFormatAnnotationForExtension(fileExtension); + if(annotation!=null) { + isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); + } + return Optional.of(createDataFileAsset(datasetCode, isaFile)); + } + return Optional.empty(); + } + + public List createAssets(List filesInDataset, String assetType, + List assays) + throws IOException, URISyntaxException, InterruptedException { + List result = new ArrayList<>(); + for(DataSetFile file : filesInDataset) { + if(!file.getPath().isBlank() && !file.isDirectory()) { + File f = new File(file.getPath()); + String datasetCode = file.getDataSetPermId().toString(); + String assetName = datasetCode+": "+f.getName(); + ISADataFile isaFile = new ISADataFile(assetName, file.getPath(), + Arrays.asList("1")); + isaFile.withAssays(assays); + String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); + String annotation = translator.dataFormatAnnotationForExtension(fileExtension); + if(annotation!=null) { + isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); + } + result.add(createDataFileAsset(datasetCode, isaFile)); + } + } + return result; + } + + public String listAssays() throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/assays/"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + return response.body(); + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + private String searchNodeWithTitle(String nodeType, String title) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/search/"; + URIBuilder builder = new URIBuilder(endpoint); + builder.setParameter("q", title).setParameter("type", "nodeType"); + + HttpRequest request = HttpRequest.newBuilder() + .uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode hits = rootNode.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + if(hit.get("title").asText().equals(title)) { + return hit.get("id").asText(); + } + } + throw new RuntimeException("Matching "+nodeType+" title was not found : " + title); + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + /** + * Searches for assays containing a search term and returns a list of found assay ids + * @param searchTerm the search term that should be in the assay properties - e.g. an openBIS id + * @return + * @throws URISyntaxException + * @throws IOException + * @throws InterruptedException + */ + public List searchAssaysContainingKeyword(String searchTerm) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/search/"; + URIBuilder builder = new URIBuilder(endpoint); + builder.setParameter("q", searchTerm).setParameter("type", "assays"); + + HttpRequest request = HttpRequest.newBuilder() + .uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode hits = rootNode.path("data"); + List assayIDs = new ArrayList<>(); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + assayIDs.add(hit.get("id").asText()); + } + return assayIDs; + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + public String updateNode(SeekStructure nodeWithChildren, String assayID, boolean transferData) { + //updateAssay(nodeWithChildren.getAssay()); + return assayID; + } + + public List createNode(SeekStructure nodeWithChildren, boolean transferData) + throws URISyntaxException, IOException, InterruptedException { + String assayID = addAssay(nodeWithChildren.getAssay()); + for(ISASample sample : nodeWithChildren.getSamples()) { + createSample(sample); + } + + List assets = new ArrayList<>(); + + Map isaToFileMap = nodeWithChildren.getIsaToOpenBISFiles(); + for(ISADataFile isaFile : isaToFileMap.keySet()) { + String assetType = nodeWithChildren.getAssetType(isaFile); + createAssetForFile(isaToFileMap.get(isaFile), assetType, Arrays.asList(assayID)) + .ifPresent(assets::add); + } + + return assets; + } + + public static class AssetToUpload { + + private final String blobEndpoint; + private final String filePath; + private final String openBISDataSetCode; + + public AssetToUpload(String blobEndpoint, String filePath, String openBISDataSetCode) { + this.blobEndpoint = blobEndpoint; + this.filePath = filePath; + this.openBISDataSetCode = openBISDataSetCode; + } + + public String getFilePath() { + return filePath; + } + + public String getBlobEndpoint() { + return blobEndpoint; + } + + public String getDataSetCode() { + return openBISDataSetCode; + } + } +} diff --git a/src/main/java/life/qbic/model/isa/AbstractISAObject.java b/src/main/java/life/qbic/model/isa/AbstractISAObject.java new file mode 100644 index 0000000..2d68900 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/AbstractISAObject.java @@ -0,0 +1,22 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.module.SimpleModule; + +public abstract class AbstractISAObject { + + public String toJson(SimpleModule module) throws JsonProcessingException { + ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(module); + ObjectWriter ow = mapper.writer().withDefaultPrettyPrinter(); + String json = ow.writeValueAsString(this); + StringBuilder jsonBuilder = new StringBuilder(); + jsonBuilder.append("{\"data\":"); + jsonBuilder.append(json); + jsonBuilder.append("}"); + return jsonBuilder.toString(); + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISAAssay.java b/src/main/java/life/qbic/model/isa/ISAAssay.java new file mode 100644 index 0000000..188d544 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISAAssay.java @@ -0,0 +1,257 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Model class for ISA Assays. Contains all mandatory and some optional properties and attributes + * that are needed to create assays in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISAAssay extends AbstractISAObject { + + private final String ISA_TYPE = "assays"; + + private Attributes attributes; + private Relationships relationships; + + public ISAAssay(String title, String studyId, String assayClass, URI assayType) { + this.attributes = new Attributes(title, assayClass, assayType); + this.relationships = new Relationships(studyId); + } + + public ISAAssay withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public ISAAssay withTags(List tags) { + this.attributes.tags = tags; + return this; + } + + public ISAAssay withDescription(String description) { + this.attributes.description = description; + return this; + } + + public ISAAssay withTechnologyType(String technologyType) { + this.attributes.technologyType = technologyType; + return this; + } + + public void setCreatorIDs(List creators) { + this.relationships.setCreatorIDs(creators); + } + public void setOrganismIDs(List organismIDs) { + this.relationships.setOrganismIDs(organismIDs); + } + public void setSampleIDs(List sampleIDs) { + this.relationships.setSampleIDs(sampleIDs); + } + public void setDataFileIDs(List dataFileIDs) { + this.relationships.setDataFileIDs(dataFileIDs); + } + public void setSOPIDs(List sopiDs) { + this.relationships.setSOPIDs(sopiDs); + } + public void setDocumentIDs(List documentIDs) { + this.relationships.setDocumentIDs(documentIDs); + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException, URISyntaxException { + ISAAssay assay = new ISAAssay("title", "1", + "EXP", + new URI("http://jermontology.org/ontology/JERMOntology#RNA-Seq")); + assay.setCreatorIDs(Arrays.asList(3,2)); + assay.setOrganismIDs(Arrays.asList(123,3332)); + System.err.println(assay.toJson()); + } + + private class Relationships { + + private String studyId; + private List creators = new ArrayList<>(); + private List samples = new ArrayList<>(); + private List documents = new ArrayList<>(); + private List dataFiles = new ArrayList<>(); + private List sops = new ArrayList<>(); + private List organisms = new ArrayList<>(); + + public Relationships(String studyId) { + this.studyId = studyId; + } + + public String getStudyId() { + return studyId; + } + + public List getCreators() { + return creators; + } + + public void setCreatorIDs(List creators) { + this.creators = creators; + } + public void setSampleIDs(List samples) { + this.samples = samples; + } + public void setDocumentIDs(List documents) { + this.documents = documents; + } + public void setDataFileIDs(List dataFiles) { + this.dataFiles = dataFiles; + } + public void setSOPIDs(List sops) { + this.sops = sops; + } + public void setOrganismIDs(List organisms) { + this.organisms = organisms; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeObjectFieldStart("study"); + jsonGenerator.writeObjectFieldStart("data"); + jsonGenerator.writeStringField("id", relationships.getStudyId()); + jsonGenerator.writeStringField("type", "studies"); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + generateListJSON(jsonGenerator, "creators", relationships.getCreators(), "people"); + generateListJSON(jsonGenerator, "samples", relationships.samples, "samples"); + generateListJSON(jsonGenerator, "documents", relationships.documents, "documents"); + generateListJSON(jsonGenerator, "data_files", relationships.dataFiles, "data_files"); + generateListJSON(jsonGenerator, "sops", relationships.sops, "sops"); + generateListJSON(jsonGenerator, "organisms", relationships.organisms, "organisms"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for(int item : items) { + generator.writeStartObject(); + generator.writeStringField("id", Integer.toString(item)); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + public List tags = new ArrayList<>(); + public String description = ""; + public String technologyType = ""; + private String title; + private AssayClass assayClass; + private AssayType assayType; + private String otherCreators = ""; + + public Attributes(String title, String assayClass, URI assayType) { + this.title = title; + this.assayClass = new AssayClass(assayClass); + this.assayType = new AssayType(assayType.toString()); + } + + public List getTags() { + return tags; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public String getTechnologyType() { + return technologyType; + } + + public AssayClass getAssay_class() { + return assayClass; + } + + public AssayType getAssay_type() { + return assayType; + } + + public String getOther_creators() { + return otherCreators; + } + + private class AssayClass { + + String key; + + public AssayClass(String assayClass) { + this.key = assayClass; + } + + public String getKey() { + return key; + } + } + + private class AssayType { + + String uri; + + public AssayType(String assayType) { + this.uri = assayType; + } + public String getUri() { + return uri; + } + } + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISADataFile.java b/src/main/java/life/qbic/model/isa/ISADataFile.java new file mode 100644 index 0000000..f2fc346 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISADataFile.java @@ -0,0 +1,203 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Model class for ISA Data files. Contains all mandatory and some optional properties and attributes + * that are needed to create a data file in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISADataFile extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private final String ISA_TYPE = "data_files"; + + public ISADataFile(String title, String fileName, List projectIds) { + this.attributes = new Attributes(title, fileName); + this.relationships = new Relationships(projectIds); + } + + public ISADataFile withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public ISADataFile withAssays(List assays) { + this.relationships.assays = assays; + return this; + } + + public ISADataFile withDataFormatAnnotations(List identifiers) { + this.attributes.withDataFormatAnnotations(identifiers); + return this; + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException { + ISADataFile sample = new ISADataFile("my file", "myfile.pdf", Arrays.asList("1")); + System.err.println(sample.toJson()); + } + + public String getFileName() { + return attributes.getContent_blobs().get(0).getOriginal_filename(); + } + + private class Relationships { + + private List projects; + private List assays; + + public Relationships(List projects) { + this.projects = projects; + this.assays = new ArrayList<>(); + } + + public List getProjects() { + return projects; + } + + public List getAssays() { + return assays; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + + generateListJSON(jsonGenerator, "projects", relationships.projects, "projects"); + generateListJSON(jsonGenerator, "assays", relationships.assays, "assays"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, + String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for (String item : items) { + generator.writeStartObject(); + generator.writeStringField("id", item); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + private String title; + private List contentBlobs = new ArrayList<>(); + private String otherCreators = ""; + private List dataFormatAnnotations = new ArrayList<>(); + + + public Attributes(String title, String fileName) { + this.title = title; + this.contentBlobs.add(new ContentBlob(fileName)); + } + + public String getTitle() { + return title; + } + + public List getContent_blobs() { + return contentBlobs; + } + + public String getOther_creators() { + return otherCreators; + } + + public List getData_format_annotations() { + return dataFormatAnnotations; + } + + public void withDataFormatAnnotations(List identifiers) { + List annotations = new ArrayList<>(); + for(String id : identifiers) { + annotations.add(new DataFormatAnnotation(id)); + } + this.dataFormatAnnotations = annotations; + } + + private class DataFormatAnnotation { + + private String label; + private String identifier; + + public DataFormatAnnotation(String identifier) { + this.identifier = identifier; + } + + public String getIdentifier() { + return identifier; + } + + } + + private class ContentBlob { + + private String originalFilename; + private String contentType; + + public ContentBlob(String fileName) { + this.originalFilename = fileName; + String suffix = fileName.substring(fileName.indexOf('.') + 1); + + this.contentType = "application/" + suffix; + } + + public String getContent_type() { + return contentType; + } + + public String getOriginal_filename() { + return originalFilename; + } + } + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISASample.java b/src/main/java/life/qbic/model/isa/ISASample.java new file mode 100644 index 0000000..5dce2de --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISASample.java @@ -0,0 +1,168 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Model class for ISA Samples. Contains all mandatory and some optional properties and attributes + * that are needed to create samples in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISASample extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private final String ISA_TYPE = "samples"; + + public ISASample(Map attributeMap, String sampleType, List projectIds) { + this.attributes = new Attributes(attributeMap); + this.relationships = new Relationships(sampleType, projectIds); + } + + public ISASample withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public void setCreatorIDs(List creatorIDs) { + this.relationships.setCreatorIDs(creatorIDs); + } + + public void setAssayIDs(List assayIDs) { + this.relationships.setAssayIDs(assayIDs); + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException { + ISASample sample = new ISASample(new HashMap<>(), "1", Arrays.asList("1")); + sample.setCreatorIDs(Arrays.asList("3","2")); + System.err.println(sample.toJson()); + } + + private class Relationships { + + private String sampleType; + private List projects; + private List creators = new ArrayList<>(); + private List assays = new ArrayList<>(); + + public Relationships(String sampleType, List projects) { + this.projects = projects; + this.sampleType = sampleType; + } + + public String getSample_type() { + return sampleType; + } + + public List getAssays() { + return assays; + } + + public void setAssayIDs(List assays) { + this.assays = assays; + } + + public List getProjects() { + return projects; + } + + public List getCreators() { + return creators; + } + + public void setCreatorIDs(List creators) { + this.creators = creators; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeObjectFieldStart("sample_type"); + jsonGenerator.writeObjectFieldStart("data"); + jsonGenerator.writeStringField("id", relationships.sampleType); + jsonGenerator.writeStringField("type", "sample_types"); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + + generateListJSON(jsonGenerator, "creators", relationships.getCreators(), "people"); + generateListJSON(jsonGenerator, "projects", relationships.projects, "projects"); + generateListJSON(jsonGenerator, "assays", relationships.assays, "assays"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for(String item : items) { + generator.writeStartObject(); + generator.writeStringField("id", item); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + private Map attributeMap = new HashMap<>(); + private String otherCreators = ""; + + public Attributes(Map attributeMap) { + this.attributeMap = attributeMap; + } + + public Map getAttribute_map() { + return attributeMap; + } + + public String getOther_creators() { + return otherCreators; + } + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISAStudy.java b/src/main/java/life/qbic/model/isa/ISAStudy.java new file mode 100644 index 0000000..1fd19bf --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISAStudy.java @@ -0,0 +1,158 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Model class for ISA Studies. Contains all mandatory and some optional properties and attributes + * that are needed to create studies in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISAStudy extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private final String ISA_TYPE = "studies"; + + public ISAStudy(String title, String investigationId) { + this.attributes = new Attributes(title); + this.relationships = new Relationships(investigationId); + } + + public ISAStudy withDescription(String description) { + this.attributes.description = description; + return this; + } + + public ISAStudy withExperimentalists(String experimentalists) { + this.attributes.experimentalists = experimentalists; + return this; + } + + public ISAStudy withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public void setCreatorIDs(List creators) { + this.relationships.setCreatorIDs(creators); + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException { + ISAStudy study = new ISAStudy("title", "1"); + study.setCreatorIDs(Arrays.asList(3,2)); + System.err.println(study.toJson()); + } + + private class Relationships { + + private String investigationId; + private List creators = new ArrayList<>(); + + public Relationships(String investigationId) { + this.investigationId = investigationId; + } + + public String getInvestigationId() { + return investigationId; + } + + public List getCreators() { + return creators; + } + + public void setCreatorIDs(List creators) { + this.creators = creators; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeObjectFieldStart("investigation"); + jsonGenerator.writeObjectFieldStart("data"); + jsonGenerator.writeStringField("id", relationships.getInvestigationId()); + jsonGenerator.writeStringField("type", "investigations"); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + jsonGenerator.writeObjectFieldStart("creators"); + jsonGenerator.writeArrayFieldStart("data"); + for(int personID : relationships.getCreators()) { + jsonGenerator.writeStartObject(); + jsonGenerator.writeNumberField("id", personID); + jsonGenerator.writeStringField("type", "people"); + jsonGenerator.writeEndObject(); + } + jsonGenerator.writeEndArray(); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + } + } + + private class Attributes { + + private String title; + private String description = ""; + private String experimentalists = ""; + private String otherCreators = ""; + + public Attributes(String title) { + this.title = title; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public String getExperimentalists() { + return experimentalists; + } + + public String getOther_creators() { + return otherCreators; + } + } + +}