From 4d28df98854fd7108942e52f421d26593139739a Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Thu, 26 Sep 2024 17:30:29 +0200 Subject: [PATCH 1/4] wip --- .../io/commandline/AuthenticationOptions.java | 85 +++- .../io/commandline/CommandLineOptions.java | 10 +- .../TransferDataToSeekCommand.java | 244 ++++++++++- .../io/commandline/UploadDatasetCommand.java | 20 +- .../OpenbisExperimentWithDescendants.java | 40 ++ .../qbic/model/OpenbisSeekTranslator.java | 121 ++++++ .../java/life/qbic/model/SeekStructure.java | 34 ++ .../qbic/model/download/OpenbisConnector.java | 71 +++- .../qbic/model/download/SEEKConnector.java | 397 ++++++++++++++++++ .../qbic/model/isa/AbstractISAObject.java | 22 + .../java/life/qbic/model/isa/ISAAssay.java | 257 ++++++++++++ .../java/life/qbic/model/isa/ISADataFile.java | 203 +++++++++ .../java/life/qbic/model/isa/ISASample.java | 168 ++++++++ .../java/life/qbic/model/isa/ISAStudy.java | 158 +++++++ 14 files changed, 1785 insertions(+), 45 deletions(-) create mode 100644 src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java create mode 100644 src/main/java/life/qbic/model/OpenbisSeekTranslator.java create mode 100644 src/main/java/life/qbic/model/SeekStructure.java create mode 100644 src/main/java/life/qbic/model/download/SEEKConnector.java create mode 100644 src/main/java/life/qbic/model/isa/AbstractISAObject.java create mode 100644 src/main/java/life/qbic/model/isa/ISAAssay.java create mode 100644 src/main/java/life/qbic/model/isa/ISADataFile.java create mode 100644 src/main/java/life/qbic/model/isa/ISASample.java create mode 100644 src/main/java/life/qbic/model/isa/ISAStudy.java diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java index 6809a06..1b9c5d0 100644 --- a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java +++ b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java @@ -23,7 +23,9 @@ public class AuthenticationOptions { description = "openBIS user name") private String openbisUser; @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. - PasswordOptions openbisPasswordOptions; + OpenbisPasswordOptions openbisPasswordOptions; + @ArgGroup(multiplicity = "1") + SeekPasswordOptions seekPasswordOptions; @Option( names = {"-as", "-as_url"}, @@ -32,17 +34,21 @@ public class AuthenticationOptions { private String as_url; @Option( - names = {"-dss", "-dss_url"}, + names = {"-dss", "--dss_url"}, description = "DatastoreServer URL", scope = CommandLine.ScopeType.INHERIT) private String dss_url; @Option( - names = {"-config", "-config_file"}, - description = "Config file path to provide openbis server information.", + names = {"-config", "--config_file"}, + description = "Config file path to provide server and user information.", scope = CommandLine.ScopeType.INHERIT) public String configPath; - + @Option( + names = {"-su", "--seek-user"}, + description = "Seek user name (email)", + scope = CommandLine.ScopeType.INHERIT) + private String seekUser; @Option( names = {"-seek-server", "-seek_url"}, description = "SEEK API URL", @@ -52,13 +58,30 @@ public class AuthenticationOptions { public String getOpenbisUser() { if(openbisUser == null & configPath!=null && !configPath.isBlank()) { openbisUser = ReadProperties.getProperties(configPath).get("user"); + } else { + log.error("No openBIS user provided."); + System.exit(2); } return openbisUser; } + public String getSeekUser() { + if(seekUser == null & configPath!=null && !configPath.isBlank()) { + seekUser = ReadProperties.getProperties(configPath).get("seek_user"); + } else { + log.error("No SEEK user/email provided."); + System.exit(2); + } + return seekUser; + } + public String getSeekURL() { - log.error("No URL to the SEEK address provided."); - System.exit(2); + if(seek_url == null & configPath!=null && !configPath.isBlank()) { + seek_url = ReadProperties.getProperties(configPath).get("seek_url"); + } else { + log.error("No URL to the SEEK address provided."); + System.exit(2); + } return seek_url; } @@ -76,6 +99,10 @@ public String getAS() { return as_url; } + public char[] getSeekPassword() { + return seekPasswordOptions.getPassword(); + } + public char[] getOpenbisPassword() { return openbisPasswordOptions.getPassword(); } @@ -83,14 +110,14 @@ public char[] getOpenbisPassword() { /** * official picocli documentation example */ - static class PasswordOptions { - @Option(names = "--password:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") + static class OpenbisPasswordOptions { + @Option(names = "--openbis-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") protected String passwordEnvironmentVariable = ""; - @Option(names = "--password:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") + @Option(names = "--openbis-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") protected String passwordProperty = ""; - @Option(names = "--password", arity = "0", description = "please provide your password", interactive = true) + @Option(names = "--openbis-pw", arity = "0", description = "please provide your openBIS password", interactive = true) protected char[] password = null; /** @@ -116,9 +143,43 @@ char[] getPassword() { System.exit(2); return null; // not reachable due to System.exit in previous line } - } + static class SeekPasswordOptions { + @Option(names = "--seek-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") + protected String passwordEnvironmentVariable = ""; + + @Option(names = "--seek-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") + protected String passwordProperty = ""; + + @Option(names = "--seek-pw", arity = "0", description = "please provide your SEEK password", interactive = true) + protected char[] password = null; + + /** + * Gets the password. If no password is provided, the program exits. + * @return the password provided by the user. + */ + char[] getPassword() { + if (nonNull(password)) { + return password; + } + // System.getProperty(String key) does not work for empty or blank keys. + if (!passwordProperty.isBlank()) { + String systemProperty = System.getProperty(passwordProperty); + if (nonNull(systemProperty)) { + return systemProperty.toCharArray(); + } + } + String environmentVariable = System.getenv(passwordEnvironmentVariable); + if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { + return environmentVariable.toCharArray(); + } + log.error("No password provided. Please provide your password."); + System.exit(2); + return null; // not reachable due to System.exit in previous line + } + + } @Override public String toString() { return new StringJoiner(", ", AuthenticationOptions.class.getSimpleName() + "[", "]") diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index d050ebf..a0dcbad 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -8,11 +8,13 @@ // main command with format specifiers for the usage help message @Command(name = "openbis-scripts", - subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, DownloadPetabCommand.class, - UploadPetabResultCommand.class, UploadDatasetCommand.class, SpaceStatisticsCommand.class }, - description = "A client software for querying openBIS.", - mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) + subcommands = {SampleHierarchyCommand.class, FindDatasetsCommand.class, + DownloadPetabCommand.class, UploadPetabResultCommand.class, UploadDatasetCommand.class, + SpaceStatisticsCommand.class, TransferDataToSeekCommand.class}, + description = "A client software for querying openBIS.", + mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) public class CommandLineOptions { + private static final Logger LOG = LogManager.getLogger(CommandLineOptions.class); @Option(names = {"-V", "--version"}, diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index 5704b3d..ec9bdaf 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -2,35 +2,164 @@ import ch.ethz.sis.openbis.generic.OpenBIS; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.io.File; -import java.util.Collections; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; import life.qbic.App; -import life.qbic.model.DatasetWithProperties; +import life.qbic.model.OpenbisExperimentWithDescendants; +import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.SeekStructure; import life.qbic.model.download.OpenbisConnector; +import life.qbic.model.download.SEEKConnector; +import life.qbic.model.download.SEEKConnector.AssetToUpload; +import org.apache.commons.codec.binary.Base64; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @Command(name = "openbis-to-seek", - description = "Transfers data or metadata from openBIS to SEEK.") + description = + "Transfers metadata and (optionally) data from openBIS to SEEK. Experiments, samples and " + + "dataset information are always transferred together (as assays, samples and one of " + + "several data types in SEEK). Dataset info always links back to the openBIS path of " + + "the respective dataset. The data itself will can transferred and stored in SEEK " + + "using the '-d' flag." + + "To completely exclude some dataset information from being transferred, a " + + "file ('--blacklist') containing dataset codes (from openBIS) can be specified." + + "Unless otherwise specified ('--no-update' flag), the command will try to update " + + "existing nodes in SEEK (recognized by openBIS identifiers in their metadata).") public class TransferDataToSeekCommand implements Runnable { - @Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset (or its metadata) to transfer. Can be found via list-data.") - private String datasetCode; - @Parameters(arity = "1", paramLabel = "seek node", description = "The node in SEEK to which to transfer the dataset.") + @Parameters(arity = "1", paramLabel = "openbis id", description = "The identifier of the " + + "experiment, sample or dataset to transfer.") + private String objectID; + @Option(names = "--blacklist", description = "Path to file specifying by dataset " + + "dataset code which openBIS datasets not to transfer to SEEK. The file must contain one code " + + "per line.") + private String blacklistFile; + @Option(names = "--no-update", description = "Use to specify that existing " + + "information in SEEK for the specified openBIS input should not be updated, but new nodes " + + "created.") + private boolean noUpdate; + /*@Option(names = {"-sn", "--seek-node"}, paramLabel = "seek node", description = + "The target node in SEEK to transfer to. Must correspond to " + + "the type of oopenBIS identifier chosen: experiment - assay; sample - sample; dataset - any of the data types. If no node is specified, " + + "a new data structure will be created in SEEK, starting from the related experiment.") private String seekNode; - @Option(names = { "-d", "--data"}, usageHelp = true, description = "Transfers the data itself to SEEK along with the metadata") + */ + @Option(names = {"-d", "--data"}, description = + "Transfers the data itself to SEEK along with the metadata. " + + "Otherwise only the link(s) to the openBIS object will be created in SEEK.") private boolean transferData; @Mixin AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisConnector openbis; + SEEKConnector seek; + OpenbisSeekTranslator translator = new OpenbisSeekTranslator(); - @Override - public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); - OpenbisConnector openbis = new OpenbisConnector(authentication); + @Override + public void run() { + System.out.println("auth..."); + + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getAS(), auth.getDSS()); + System.out.println("openbis..."); + + openbis = new OpenbisConnector(authentication); + + boolean isSample = false; + boolean isDataSet = false; + System.out.println("search for experiment..."); + + boolean isExperiment = experimentExists(objectID); + if (!isExperiment && sampleExists(objectID)) { + isSample = true; + } + + if (!isExperiment && !isSample && datasetsExist(Arrays.asList(objectID))) { + isDataSet = true; + } + + if (!isSample && !isExperiment && !isDataSet) { + System.out.printf( + "%s could not be found in openBIS. Make sure you either specify an experiment, sample or dataset%n", + objectID); + return; + } + System.out.println("Searching done..."); +/* + if (isExperiment) { + if (seekNode != null && !seekNode.contains("assays")) { + System.out.printf( + "Seek node %s does not correspond to the provided openBIS experiment identifier. " + + "Please select an assay node.%n", objectID); + return; + } + } + if (isSample) { + if (seekNode != null && !seekNode.contains("samples")) { + System.out.printf( + "Seek node %s does not correspond to the provided openBIS sample identifier. " + + "Please select a sample node.%n", objectID); + return; + } + } + + */ + + byte[] httpCredentials = Base64.encodeBase64( + (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); + try { + seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "Default Project", + "lisym default study"); + } catch (URISyntaxException | IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + /* + if (seekNode != null) { + try { + if (!seek.endPointExists(seekNode)) { + System.out.println(seekNode + " could not be found"); + return; + } else { + if (isExperiment) { + Experiment experimentWithSamplesAndDatasets = openbis.getExperimentWithDescendants( + objectID); + //SeekStructure seekStructure = translator.translate(seekNode, experimentWithSamplesAndDatasets); + //seek.fillAssayWithSamplesAndDatasets(seekNode, seekStructure); + } + } + } catch (URISyntaxException | InterruptedException | IOException e) { + throw new RuntimeException(e); + } + } + + */ + try { + OpenbisExperimentWithDescendants experiment = openbis.getExperimentWithDescendants(objectID); + String assayID = getAssayIDForOpenBISExperiment(experiment.getExperiment()); + List blacklist = parseBlackList(blacklistFile); + if(assayID.isBlank()) { + createNewNodes(experiment, blacklist); + } else { + updateNodes(experiment, assayID, blacklist); + } + } catch (URISyntaxException | InterruptedException | IOException e) { + throw new RuntimeException(e); + } + + /* List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); @@ -38,6 +167,7 @@ public void run() { System.out.println(datasetCode+" not found"); return; } + DatasetWithProperties result = new DatasetWithProperties(datasets.get(0)); Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", result.getExperiment().getIdentifier()); @@ -53,9 +183,97 @@ public void run() { cleanupTemp(new File(tmpPath)); +*/ + System.out.println("Done"); + } - System.out.println("Done"); + private List parseBlackList(String blacklistFile) { + List result = new ArrayList<>(); + if(blacklistFile == null) { + return result; + } + // trim whitespace, skip empty lines + try (Stream lines = Files.lines(Paths.get(blacklistFile)) + .map(String::trim) + .filter(s -> !s.isBlank())) { + return lines.collect(Collectors.toList()); + } catch (IOException e) { + throw new RuntimeException(blacklistFile+" could not be found or read."); } + } + + private void updateNodes(OpenbisExperimentWithDescendants experiment, String assayID, List blacklist) { + System.err.println("updating nodes of assay "+assayID); + SeekStructure nodeWithChildren = translator.translate(experiment, blacklist); + String updatedEndpoint = seek.updateNode(nodeWithChildren, assayID, transferData); + System.out.printf("%s was successfully updated.%n", endpoint); + } + + private void createNewNodes(OpenbisExperimentWithDescendants experiment, List blacklist) + throws URISyntaxException, IOException, InterruptedException { + System.err.println("creating new nodes"); + SeekStructure nodeWithChildren = translator.translate(experiment, blacklist); + List assetsToUpload = seek.createNode(nodeWithChildren, transferData); + if(transferData) { + for(AssetToUpload asset : assetsToUpload) { + System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); + String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), + () -> openbis.streamDataset(asset.getDataSetCode(), asset.getFilePath())); + System.out.printf("File stored here: %s%n", fileURL); + } + } + System.out.printf("%s was successfully created.%n", endpoint); + } + + private boolean sampleExists(String objectID) { + return openbis.sampleExists(objectID); + } + + private boolean datasetsExist(List datasetCodes) { + return openbis.findDataSets(datasetCodes).size() == datasetCodes.size(); + } + + private boolean experimentExists(String experimentID) { + return openbis.experimentExists(experimentID); + } + + private String getAssayIDForOpenBISExperiment(Experiment experiment) + throws URISyntaxException, IOException, InterruptedException { + // the perm id is unique and afaik not used by scientists. it is highly unlikely that it would + // "accidentally" be part of another title or description. however, care should be taken here, + // because if a perm id is found in the wrong SEEK node, meta-information in SEEK could be + // overwritten or samples/data added to the wrong assay. + String permID = experiment.getPermId().getPermId(); + List assayIDs = seek.searchAssaysContainingKeyword(permID); + if(assayIDs.isEmpty()) { + return ""; + } + if(assayIDs.size() == 1) { + return assayIDs.get(0); + } + throw new RuntimeException("Search term "+permID+ " was found in more than one assay: "+assayIDs); + } + + private void sendDatasetToSeek(String datasetCode, String assayID) + throws URISyntaxException, IOException, InterruptedException { + assayID = "3"; + System.out.println("Searching dataset in openBIS..."); + List datasets = openbis.findDataSets( + Arrays.asList(datasetCode)); + if(datasets.isEmpty()) { + return; + } + DataSet dataset = datasets.get(0); + List files = openbis.getDatasetFiles(dataset); + List assets = seek.createAssets(files, dataset.getType().getCode(), + Arrays.asList(assayID)); + for(AssetToUpload asset : assets) { + System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); + String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), + () -> openbis.streamDataset(datasetCode, asset.getFilePath())); + System.out.printf("File stored here: %s%n", fileURL); + } + } private void cleanupTemp(File tmpFolder) { File[] files = tmpFolder.listFiles(); diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index f165c9a..9e2d3a9 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -39,21 +39,13 @@ public void run() { System.out.printf("Path %s could not be found%n", dataPath); return; } - boolean attachToSample = OpenbisConnector.sampleIdPattern.matcher(objectID).find(); - boolean attachToExperiment = false; - if(!attachToSample) { - attachToExperiment = OpenbisConnector.experimentIdPattern.matcher(objectID).find(); + boolean attachToSample = false; + boolean attachToExperiment = experimentExists(objectID); + if(sampleExists(objectID)) { + attachToSample = true; } - if(!attachToExperiment && !attachToSample) { - System.out.printf("%s is neither a valid experiment nor sample identifier%n", objectID); - return; - } - if(attachToExperiment && !experimentExists(objectID)) { - System.out.printf("Experiment with identifier %s could not be found%n", objectID); - return; - } - if(attachToSample && !sampleExists(objectID)) { - System.out.printf("Sample object with identifier %s could not be found%n", objectID); + if(!attachToSample && !attachToExperiment) { + System.out.printf("%s could not be found in openBIS.%n", objectID); return; } if(!datasetsExist(parents)) { diff --git a/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java new file mode 100644 index 0000000..86399ec --- /dev/null +++ b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java @@ -0,0 +1,40 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.util.List; +import java.util.Map; + +public class OpenbisExperimentWithDescendants { + + private Experiment experiment; + private List samples; + private List datasets; + private Map> datasetCodeToFiles; + + public OpenbisExperimentWithDescendants(Experiment experiment, List samples, + List datasets, Map> datasetCodeToFiles) { + this.experiment = experiment; + this.samples = samples; + this.datasets = datasets; + this.datasetCodeToFiles = datasetCodeToFiles; + } + + public Experiment getExperiment() { + return experiment; + } + + public List getSamples() { + return samples; + } + + public List getDatasets() { + return datasets; + } + + public List getFilesForDataset(String permID) { + return datasetCodeToFiles.get(permID); + } +} diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java new file mode 100644 index 0000000..a02317d --- /dev/null +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -0,0 +1,121 @@ +package life.qbic.model; + +import static java.util.Map.entry; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import life.qbic.model.isa.ISADataFile; +import life.qbic.model.isa.ISASample; + +public class OpenbisSeekTranslator { + + private final String DEFAULT_PROJECT_ID; + private final String DEFAULT_STUDY_ID; + + public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { + this.DEFAULT_PROJECT_ID = defaultProjectID; + this.DEFAULT_STUDY_ID = defaultStudyID; + } + + Map experimentTypeToAssayClass = Map.ofEntries( + entry("00_MOUSE_DATABASE", "EXP"), + entry("00_PATIENT_DATABASE", "EXP"), + entry("00_STANDARD_OPERATING_PROTOCOLS", "EXP"), + entry("01_BIOLOGICAL_EXPERIMENT", "EXP"), + entry("02_MASSSPECTROMETRY_EXPERIMENT", "EXP"),//where is petab modeling attached? + entry("03_HISTOLOGICAL_ANALYSIS", "EXP"), + entry("04_MICRO_CT", "EXP") + ); + + Map experimentTypeToAssayType = Map.ofEntries( + entry("00_MOUSE_DATABASE", ""), + entry("00_PATIENT_DATABASE", ""),//if this is related to measured data, attach it as sample to the assay + entry("00_STANDARD_OPERATING_PROTOCOLS", ""), + entry("01_BIOLOGICAL_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Cultivation_experiment"), + entry("02_MASSSPECTROMETRY_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Proteomics"), + entry("03_HISTOLOGICAL_ANALYSIS", ""), + entry("04_MICRO_CT", "") + ); + + Map fileExtensionToDataFormat = Map.ofEntries( + entry("fastq.gz", "http://edamontology.org/format_1930"), + entry("fastq", "http://edamontology.org/format_1930"), + entry("json", "http://edamontology.org/format_3464"), + entry("yaml", "http://edamontology.org/format_3750"), + entry("raw", "http://edamontology.org/format_3712"), + entry("tsv", "http://edamontology.org/format_3475"), + entry("csv", "http://edamontology.org/format_3752") + ); + + Map datasetTypeToAssetType = Map.ofEntries( + entry("ANALYSIS_NOTEBOOK", "Document"), + entry("ANALYZED_DATA", "Data_file"), + entry("ATTACHMENT", "Document"), + entry("ELN_PREVIEW", ""), + entry("EXPERIMENT_PROTOCOL", "SOP"), + entry("EXPERIMENT_RESULT", "Document"), + entry("HISTOLOGICAL_SLIDE", "Data_file"), + entry("IB_DATA", "Data_file"), + entry("LUMINEX_DATA", "Data_file"), + entry("MS_DATA_ANALYZED", "Data_file"), + entry("MS_DATA_RAW", "Data_file"), + entry("OTHER_DATA", "Document"), + entry("PROCESSED_DATA", "Data_file"), + entry("PUBLICATION_DATA", "Publication"), + entry("QPCR_DATA", "Data_file"), + entry("RAW_DATA", "Data_file"), + entry("SOURCE_CODE", "Document"), + entry("TEST_CONT", ""), + entry("TEST_DAT", ""), + entry("UNKNOWN", "") + ); + + public SeekStructure translate(String seekNode, Experiment experimentWithSamplesAndDatasets) { + Experiment exp = experimentWithSamplesAndDatasets; + exp.getType(); + return null; + //new ISAAssay(exp.getCode(), ) + } + + public ISADataFile translate(String seekNode, DataSet dataset) { + return null; //new ISADataFile(); + } + + public String assetForDatasetType(String datasetType) { + return "Data_file";//TODO + } + + public String dataFormatAnnotationForExtension(String fileExtension) { + return fileExtensionToDataFormat.get(fileExtension); + } + + public SeekStructure translate(Experiment experiment, List blacklist) { + System.err.println(experiment.getCode()); + + for(Sample sample : experiment.getSamples()) { + String sampleType = getSampleType(sample.getType()); + Map attributes = new HashMap<>(); + ISASample isaSample = new ISASample(attributes, sampleType, + Collections.singletonList(DEFAULT_PROJECT_ID)); + System.err.println(sample.getCode()); + } + for(DataSet dataset : experiment.getDataSets()) { + System.err.println(dataset.getCode()); + } + System.err.println("blacklisted:"); + for(String dataset : blacklist) { + System.err.println(dataset); + } + return null;//TODO + } + + private String getSampleType(SampleType type) { + } +} diff --git a/src/main/java/life/qbic/model/SeekStructure.java b/src/main/java/life/qbic/model/SeekStructure.java new file mode 100644 index 0000000..5d7cc46 --- /dev/null +++ b/src/main/java/life/qbic/model/SeekStructure.java @@ -0,0 +1,34 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.util.List; +import java.util.Map; +import life.qbic.model.isa.ISAAssay; +import life.qbic.model.isa.ISADataFile; +import life.qbic.model.isa.ISASample; + +public class SeekStructure { + + private ISAAssay assay; + private List samples; + private List dataFiles; + private Map isaToOpenBISFile; + private Map isaFileToAssetType; + //ISASample isaSample = new ISASample(map, sampleTypeID, Arrays.asList(projectID)); + + public ISAAssay getAssay() { + return assay; + } + + public List getSamples() { + return samples; + } + + public Map getIsaToOpenBISFiles() { + return isaToOpenBISFile; + } + + public String getAssetType(ISADataFile dataFile) { + return isaFileToAssetType.get(dataFile); + } +} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index c73d796..e50dd12 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -25,8 +25,10 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownload; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadOptions; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadReader; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.DataSetFilePermId; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.IDataSetFileId; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -40,6 +42,8 @@ import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; +import life.qbic.model.DatasetWithProperties; +import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.SampleTypeConnection; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -49,8 +53,9 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); private final OpenBIS openBIS; - public final static Pattern experimentIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); - public final static Pattern sampleIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); + //public final static Pattern experimentIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); + //public final static Pattern sampleIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); + public static Pattern datasetCodePattern = Pattern.compile("[0-9]^{17}-[0-9]+"); public OpenbisConnector(OpenBIS authentication) { this.openBIS = authentication; @@ -161,6 +166,22 @@ public File downloadDataset(String targetPath, String datasetID) { return new File(targetPath); } + public InputStream streamDataset(String datasetCode, String filePath) { + DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); + IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetCode), + filePath); + + // Setting recursive flag to true will return both subfolders and files + options.setRecursive(true); + + // Read the contents and print them out + InputStream stream = openBIS.downloadFiles(new ArrayList<>(List.of(fileToDownload)), + options); + + DataSetFileDownloadReader reader = new DataSetFileDownloadReader(stream); + return reader.read().getInputStream(); + } + public Map queryFullSampleHierarchy(List spaces) { Map hierarchy = new HashMap<>(); if (spaces.isEmpty()) { @@ -370,9 +391,14 @@ public List findDataSets(List codes) { criteria.withCodes().thatIn(codes); DataSetFetchOptions options = new DataSetFetchOptions(); options.withExperiment(); + options.withType(); return openBIS.searchDataSets(criteria, options).getObjects(); } + public boolean datasetExists(String code) { + return !findDataSets(new ArrayList<>(Arrays.asList(code))).isEmpty(); + } + public boolean experimentExists(String experimentID) { ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); criteria.withIdentifier().thatEquals(experimentID); @@ -388,4 +414,45 @@ public boolean sampleExists(String objectID) { return !openBIS.searchSamples(criteria, new SampleFetchOptions()).getObjects() .isEmpty(); } + + public OpenbisExperimentWithDescendants getExperimentWithDescendants(String experimentID) { + ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); + criteria.withIdentifier().thatEquals(experimentID); + + ExperimentFetchOptions fetchOptions = new ExperimentFetchOptions(); + fetchOptions.withType(); + fetchOptions.withProject(); + fetchOptions.withProperties(); + DataSetFetchOptions dataSetFetchOptions = new DataSetFetchOptions(); + dataSetFetchOptions.withType(); + dataSetFetchOptions.withRegistrator(); + SampleFetchOptions sampleFetchOptions = new SampleFetchOptions(); + sampleFetchOptions.withProperties(); + sampleFetchOptions.withDataSetsUsing(dataSetFetchOptions); + fetchOptions.withDataSetsUsing(dataSetFetchOptions); + fetchOptions.withSamplesUsing(sampleFetchOptions); + + Experiment experiment = openBIS.searchExperiments(criteria, fetchOptions).getObjects().get(0); + + Map> datasetCodeToFiles = new HashMap<>(); + for(DataSet dataset : experiment.getDataSets()) { + datasetCodeToFiles.put(dataset.getPermId(), getDatasetFiles(dataset)); + } + + return new OpenbisExperimentWithDescendants(experiment, experiment.getSamples(), + experiment.getDataSets() + .stream().map(DatasetWithProperties::new) + .collect(Collectors.toList()), datasetCodeToFiles); + } + + public List getDatasetFiles(DataSet dataset) { + DataSetFileSearchCriteria criteria = new DataSetFileSearchCriteria(); + + DataSetSearchCriteria dataSetCriteria = criteria.withDataSet().withOrOperator(); + dataSetCriteria.withCode().thatEquals(dataset.getCode()); + + SearchResult result = openBIS.searchFiles(criteria, new DataSetFileFetchOptions()); + + return result.getObjects(); + } } diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java new file mode 100644 index 0000000..298364e --- /dev/null +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -0,0 +1,397 @@ +package life.qbic.model.download; + +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; +import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.SeekStructure; +import life.qbic.model.isa.ISAAssay; +import life.qbic.model.isa.ISADataFile; +import life.qbic.model.isa.ISASample; +import life.qbic.model.isa.ISAStudy; +import org.apache.http.client.utils.URIBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class SEEKConnector { + + private static final Logger LOG = LogManager.getLogger(SEEKConnector.class); + private String apiURL; + private byte[] credentials; + private OpenbisSeekTranslator translator; + + public SEEKConnector(String apiURL, byte[] httpCredentials, String defaultProjectTitle, + String defaultStudyTitle) throws URISyntaxException, IOException, InterruptedException { + this.apiURL = apiURL; + this.credentials = httpCredentials; + translator = new OpenbisSeekTranslator(searchNodeWithTitle("projects", defaultProjectTitle), + searchNodeWithTitle("studies", defaultStudyTitle)); + } + + public String addAssay(ISAAssay assay) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/assays"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, assay.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + + public String createStudy(ISAStudy study) + throws IOException, URISyntaxException, InterruptedException, IOException { + String endpoint = apiURL+"/studies"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, study.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + + private HttpRequest buildAuthorizedPOSTRequest(String endpoint, String body) throws URISyntaxException { + return HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .POST(HttpRequest.BodyPublishers.ofString(body)).build(); + } + + public boolean studyExists(String id) throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/studies/"+id; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + return response.statusCode() == 200; + } + + /* + -datatype by extension +-assay equals experiment +-investigation: pre-created in SEEK? +-study: project +-patient id should be linked somehow, maybe gender? +-flexible object type to sample type? + */ + + public String createSample(ISASample isaSample) throws URISyntaxException, IOException, + InterruptedException { + String endpoint = apiURL+"/samples"; + HashMap map = new HashMap<>(); + //map.put("title", "really?"); + map.put("test_attribute", "okay then"); + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, isaSample.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=201) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + + private AssetToUpload createDataFileAsset(String datasetCode, ISADataFile data) + throws IOException, URISyntaxException, InterruptedException { + String endpoint = apiURL+"/data_files"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, data.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=201 && response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data") + .path("attributes") + .path("content_blobs") + .path(0).path("link"); + return new AssetToUpload(idNode.asText(), data.getFileName(), datasetCode); + } + + @Deprecated + private void uploadFileContent(String assetType, String assetID, String blobID, String file) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/"+assetType+"/"+assetID+"/content_blobs/"+blobID; + + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/octet-stream") + .headers("Accept", "application/octet-stream") + .headers("Authorization", "Basic " + new String(credentials)) + .PUT(BodyPublishers.ofFile(new File(file).toPath())).build(); + + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + public String uploadStreamContent(String blobEndpoint, + Supplier streamSupplier) + throws URISyntaxException, IOException, InterruptedException { + + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(blobEndpoint)) + .headers("Content-Type", "application/octet-stream") + .headers("Accept", "application/octet-stream") + .headers("Authorization", "Basic " + new String(credentials)) + .PUT(BodyPublishers.ofInputStream(streamSupplier)).build(); + + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } else { + String fileURL = blobEndpoint.split("content_blobs")[0]; + return fileURL; + } + } + + public boolean endPointExists(String endpoint) + throws URISyntaxException, IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + return response.statusCode() == 200; + } + + /** + * Creates an optional asset for a file from an openBIS dataset. Folders are ignored. + * @param file + * @param assetType + * @param assays + * @return + * @throws IOException + * @throws URISyntaxException + * @throws InterruptedException + */ + public Optional createAssetForFile(DataSetFile file, String assetType, + List assays) + throws IOException, URISyntaxException, InterruptedException { + if(!file.getPath().isBlank() && !file.isDirectory()) { + File f = new File(file.getPath()); + String datasetCode = file.getDataSetPermId().toString(); + String assetName = datasetCode+": "+f.getName(); + ISADataFile isaFile = new ISADataFile(assetName, file.getPath(), + Arrays.asList("1")); + isaFile.withAssays(assays); + String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); + String annotation = translator.dataFormatAnnotationForExtension(fileExtension); + if(annotation!=null) { + isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); + } + return Optional.of(createDataFileAsset(datasetCode, isaFile)); + } + return Optional.empty(); + } + + public List createAssets(List filesInDataset, String assetType, + List assays) + throws IOException, URISyntaxException, InterruptedException { + List result = new ArrayList<>(); + for(DataSetFile file : filesInDataset) { + if(!file.getPath().isBlank() && !file.isDirectory()) { + File f = new File(file.getPath()); + String datasetCode = file.getDataSetPermId().toString(); + String assetName = datasetCode+": "+f.getName(); + ISADataFile isaFile = new ISADataFile(assetName, file.getPath(), + Arrays.asList("1")); + isaFile.withAssays(assays); + String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); + String annotation = translator.dataFormatAnnotationForExtension(fileExtension); + if(annotation!=null) { + isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); + } + result.add(createDataFileAsset(datasetCode, isaFile)); + } + } + return result; + } + + public String listAssays() throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/assays/"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + return response.body(); + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + private String searchNodeWithTitle(String nodeType, String title) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/search/"; + URIBuilder builder = new URIBuilder(endpoint); + builder.setParameter("q", title).setParameter("type", "nodeType"); + + HttpRequest request = HttpRequest.newBuilder() + .uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode hits = rootNode.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + if(hit.get("title").asText().equals(title)) { + return hit.get("id").asText(); + } + } + throw new RuntimeException("Matching "+nodeType+" title was not found : " + title); + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + /** + * Searches for assays containing a search term and returns a list of found assay ids + * @param searchTerm the search term that should be in the assay properties - e.g. an openBIS id + * @return + * @throws URISyntaxException + * @throws IOException + * @throws InterruptedException + */ + public List searchAssaysContainingKeyword(String searchTerm) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/search/"; + URIBuilder builder = new URIBuilder(endpoint); + builder.setParameter("q", searchTerm).setParameter("type", "assays"); + + HttpRequest request = HttpRequest.newBuilder() + .uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode hits = rootNode.path("data"); + List assayIDs = new ArrayList<>(); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + assayIDs.add(hit.get("id").asText()); + } + return assayIDs; + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + public String updateNode(SeekStructure nodeWithChildren, String assayID, boolean transferData) { + //updateAssay(nodeWithChildren.getAssay()); + return assayID; + } + + public List createNode(SeekStructure nodeWithChildren, boolean transferData) + throws URISyntaxException, IOException, InterruptedException { + String assayID = addAssay(nodeWithChildren.getAssay()); + for(ISASample sample : nodeWithChildren.getSamples()) { + createSample(sample); + } + + List assets = new ArrayList<>(); + + Map isaToFileMap = nodeWithChildren.getIsaToOpenBISFiles(); + for(ISADataFile isaFile : isaToFileMap.keySet()) { + String assetType = nodeWithChildren.getAssetType(isaFile); + createAssetForFile(isaToFileMap.get(isaFile), assetType, Arrays.asList(assayID)) + .ifPresent(assets::add); + } + + return assets; + } + + public static class AssetToUpload { + + private final String blobEndpoint; + private final String filePath; + private final String openBISDataSetCode; + + public AssetToUpload(String blobEndpoint, String filePath, String openBISDataSetCode) { + this.blobEndpoint = blobEndpoint; + this.filePath = filePath; + this.openBISDataSetCode = openBISDataSetCode; + } + + public String getFilePath() { + return filePath; + } + + public String getBlobEndpoint() { + return blobEndpoint; + } + + public String getDataSetCode() { + return openBISDataSetCode; + } + } +} diff --git a/src/main/java/life/qbic/model/isa/AbstractISAObject.java b/src/main/java/life/qbic/model/isa/AbstractISAObject.java new file mode 100644 index 0000000..2d68900 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/AbstractISAObject.java @@ -0,0 +1,22 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.module.SimpleModule; + +public abstract class AbstractISAObject { + + public String toJson(SimpleModule module) throws JsonProcessingException { + ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(module); + ObjectWriter ow = mapper.writer().withDefaultPrettyPrinter(); + String json = ow.writeValueAsString(this); + StringBuilder jsonBuilder = new StringBuilder(); + jsonBuilder.append("{\"data\":"); + jsonBuilder.append(json); + jsonBuilder.append("}"); + return jsonBuilder.toString(); + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISAAssay.java b/src/main/java/life/qbic/model/isa/ISAAssay.java new file mode 100644 index 0000000..188d544 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISAAssay.java @@ -0,0 +1,257 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Model class for ISA Assays. Contains all mandatory and some optional properties and attributes + * that are needed to create assays in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISAAssay extends AbstractISAObject { + + private final String ISA_TYPE = "assays"; + + private Attributes attributes; + private Relationships relationships; + + public ISAAssay(String title, String studyId, String assayClass, URI assayType) { + this.attributes = new Attributes(title, assayClass, assayType); + this.relationships = new Relationships(studyId); + } + + public ISAAssay withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public ISAAssay withTags(List tags) { + this.attributes.tags = tags; + return this; + } + + public ISAAssay withDescription(String description) { + this.attributes.description = description; + return this; + } + + public ISAAssay withTechnologyType(String technologyType) { + this.attributes.technologyType = technologyType; + return this; + } + + public void setCreatorIDs(List creators) { + this.relationships.setCreatorIDs(creators); + } + public void setOrganismIDs(List organismIDs) { + this.relationships.setOrganismIDs(organismIDs); + } + public void setSampleIDs(List sampleIDs) { + this.relationships.setSampleIDs(sampleIDs); + } + public void setDataFileIDs(List dataFileIDs) { + this.relationships.setDataFileIDs(dataFileIDs); + } + public void setSOPIDs(List sopiDs) { + this.relationships.setSOPIDs(sopiDs); + } + public void setDocumentIDs(List documentIDs) { + this.relationships.setDocumentIDs(documentIDs); + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException, URISyntaxException { + ISAAssay assay = new ISAAssay("title", "1", + "EXP", + new URI("http://jermontology.org/ontology/JERMOntology#RNA-Seq")); + assay.setCreatorIDs(Arrays.asList(3,2)); + assay.setOrganismIDs(Arrays.asList(123,3332)); + System.err.println(assay.toJson()); + } + + private class Relationships { + + private String studyId; + private List creators = new ArrayList<>(); + private List samples = new ArrayList<>(); + private List documents = new ArrayList<>(); + private List dataFiles = new ArrayList<>(); + private List sops = new ArrayList<>(); + private List organisms = new ArrayList<>(); + + public Relationships(String studyId) { + this.studyId = studyId; + } + + public String getStudyId() { + return studyId; + } + + public List getCreators() { + return creators; + } + + public void setCreatorIDs(List creators) { + this.creators = creators; + } + public void setSampleIDs(List samples) { + this.samples = samples; + } + public void setDocumentIDs(List documents) { + this.documents = documents; + } + public void setDataFileIDs(List dataFiles) { + this.dataFiles = dataFiles; + } + public void setSOPIDs(List sops) { + this.sops = sops; + } + public void setOrganismIDs(List organisms) { + this.organisms = organisms; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeObjectFieldStart("study"); + jsonGenerator.writeObjectFieldStart("data"); + jsonGenerator.writeStringField("id", relationships.getStudyId()); + jsonGenerator.writeStringField("type", "studies"); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + generateListJSON(jsonGenerator, "creators", relationships.getCreators(), "people"); + generateListJSON(jsonGenerator, "samples", relationships.samples, "samples"); + generateListJSON(jsonGenerator, "documents", relationships.documents, "documents"); + generateListJSON(jsonGenerator, "data_files", relationships.dataFiles, "data_files"); + generateListJSON(jsonGenerator, "sops", relationships.sops, "sops"); + generateListJSON(jsonGenerator, "organisms", relationships.organisms, "organisms"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for(int item : items) { + generator.writeStartObject(); + generator.writeStringField("id", Integer.toString(item)); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + public List tags = new ArrayList<>(); + public String description = ""; + public String technologyType = ""; + private String title; + private AssayClass assayClass; + private AssayType assayType; + private String otherCreators = ""; + + public Attributes(String title, String assayClass, URI assayType) { + this.title = title; + this.assayClass = new AssayClass(assayClass); + this.assayType = new AssayType(assayType.toString()); + } + + public List getTags() { + return tags; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public String getTechnologyType() { + return technologyType; + } + + public AssayClass getAssay_class() { + return assayClass; + } + + public AssayType getAssay_type() { + return assayType; + } + + public String getOther_creators() { + return otherCreators; + } + + private class AssayClass { + + String key; + + public AssayClass(String assayClass) { + this.key = assayClass; + } + + public String getKey() { + return key; + } + } + + private class AssayType { + + String uri; + + public AssayType(String assayType) { + this.uri = assayType; + } + public String getUri() { + return uri; + } + } + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISADataFile.java b/src/main/java/life/qbic/model/isa/ISADataFile.java new file mode 100644 index 0000000..f2fc346 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISADataFile.java @@ -0,0 +1,203 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Model class for ISA Data files. Contains all mandatory and some optional properties and attributes + * that are needed to create a data file in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISADataFile extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private final String ISA_TYPE = "data_files"; + + public ISADataFile(String title, String fileName, List projectIds) { + this.attributes = new Attributes(title, fileName); + this.relationships = new Relationships(projectIds); + } + + public ISADataFile withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public ISADataFile withAssays(List assays) { + this.relationships.assays = assays; + return this; + } + + public ISADataFile withDataFormatAnnotations(List identifiers) { + this.attributes.withDataFormatAnnotations(identifiers); + return this; + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException { + ISADataFile sample = new ISADataFile("my file", "myfile.pdf", Arrays.asList("1")); + System.err.println(sample.toJson()); + } + + public String getFileName() { + return attributes.getContent_blobs().get(0).getOriginal_filename(); + } + + private class Relationships { + + private List projects; + private List assays; + + public Relationships(List projects) { + this.projects = projects; + this.assays = new ArrayList<>(); + } + + public List getProjects() { + return projects; + } + + public List getAssays() { + return assays; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + + generateListJSON(jsonGenerator, "projects", relationships.projects, "projects"); + generateListJSON(jsonGenerator, "assays", relationships.assays, "assays"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, + String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for (String item : items) { + generator.writeStartObject(); + generator.writeStringField("id", item); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + private String title; + private List contentBlobs = new ArrayList<>(); + private String otherCreators = ""; + private List dataFormatAnnotations = new ArrayList<>(); + + + public Attributes(String title, String fileName) { + this.title = title; + this.contentBlobs.add(new ContentBlob(fileName)); + } + + public String getTitle() { + return title; + } + + public List getContent_blobs() { + return contentBlobs; + } + + public String getOther_creators() { + return otherCreators; + } + + public List getData_format_annotations() { + return dataFormatAnnotations; + } + + public void withDataFormatAnnotations(List identifiers) { + List annotations = new ArrayList<>(); + for(String id : identifiers) { + annotations.add(new DataFormatAnnotation(id)); + } + this.dataFormatAnnotations = annotations; + } + + private class DataFormatAnnotation { + + private String label; + private String identifier; + + public DataFormatAnnotation(String identifier) { + this.identifier = identifier; + } + + public String getIdentifier() { + return identifier; + } + + } + + private class ContentBlob { + + private String originalFilename; + private String contentType; + + public ContentBlob(String fileName) { + this.originalFilename = fileName; + String suffix = fileName.substring(fileName.indexOf('.') + 1); + + this.contentType = "application/" + suffix; + } + + public String getContent_type() { + return contentType; + } + + public String getOriginal_filename() { + return originalFilename; + } + } + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISASample.java b/src/main/java/life/qbic/model/isa/ISASample.java new file mode 100644 index 0000000..5dce2de --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISASample.java @@ -0,0 +1,168 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Model class for ISA Samples. Contains all mandatory and some optional properties and attributes + * that are needed to create samples in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISASample extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private final String ISA_TYPE = "samples"; + + public ISASample(Map attributeMap, String sampleType, List projectIds) { + this.attributes = new Attributes(attributeMap); + this.relationships = new Relationships(sampleType, projectIds); + } + + public ISASample withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public void setCreatorIDs(List creatorIDs) { + this.relationships.setCreatorIDs(creatorIDs); + } + + public void setAssayIDs(List assayIDs) { + this.relationships.setAssayIDs(assayIDs); + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException { + ISASample sample = new ISASample(new HashMap<>(), "1", Arrays.asList("1")); + sample.setCreatorIDs(Arrays.asList("3","2")); + System.err.println(sample.toJson()); + } + + private class Relationships { + + private String sampleType; + private List projects; + private List creators = new ArrayList<>(); + private List assays = new ArrayList<>(); + + public Relationships(String sampleType, List projects) { + this.projects = projects; + this.sampleType = sampleType; + } + + public String getSample_type() { + return sampleType; + } + + public List getAssays() { + return assays; + } + + public void setAssayIDs(List assays) { + this.assays = assays; + } + + public List getProjects() { + return projects; + } + + public List getCreators() { + return creators; + } + + public void setCreatorIDs(List creators) { + this.creators = creators; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeObjectFieldStart("sample_type"); + jsonGenerator.writeObjectFieldStart("data"); + jsonGenerator.writeStringField("id", relationships.sampleType); + jsonGenerator.writeStringField("type", "sample_types"); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + + generateListJSON(jsonGenerator, "creators", relationships.getCreators(), "people"); + generateListJSON(jsonGenerator, "projects", relationships.projects, "projects"); + generateListJSON(jsonGenerator, "assays", relationships.assays, "assays"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for(String item : items) { + generator.writeStartObject(); + generator.writeStringField("id", item); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + private Map attributeMap = new HashMap<>(); + private String otherCreators = ""; + + public Attributes(Map attributeMap) { + this.attributeMap = attributeMap; + } + + public Map getAttribute_map() { + return attributeMap; + } + + public String getOther_creators() { + return otherCreators; + } + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISAStudy.java b/src/main/java/life/qbic/model/isa/ISAStudy.java new file mode 100644 index 0000000..1fd19bf --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISAStudy.java @@ -0,0 +1,158 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Model class for ISA Studies. Contains all mandatory and some optional properties and attributes + * that are needed to create studies in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISAStudy extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private final String ISA_TYPE = "studies"; + + public ISAStudy(String title, String investigationId) { + this.attributes = new Attributes(title); + this.relationships = new Relationships(investigationId); + } + + public ISAStudy withDescription(String description) { + this.attributes.description = description; + return this; + } + + public ISAStudy withExperimentalists(String experimentalists) { + this.attributes.experimentalists = experimentalists; + return this; + } + + public ISAStudy withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public void setCreatorIDs(List creators) { + this.relationships.setCreatorIDs(creators); + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException { + ISAStudy study = new ISAStudy("title", "1"); + study.setCreatorIDs(Arrays.asList(3,2)); + System.err.println(study.toJson()); + } + + private class Relationships { + + private String investigationId; + private List creators = new ArrayList<>(); + + public Relationships(String investigationId) { + this.investigationId = investigationId; + } + + public String getInvestigationId() { + return investigationId; + } + + public List getCreators() { + return creators; + } + + public void setCreatorIDs(List creators) { + this.creators = creators; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeObjectFieldStart("investigation"); + jsonGenerator.writeObjectFieldStart("data"); + jsonGenerator.writeStringField("id", relationships.getInvestigationId()); + jsonGenerator.writeStringField("type", "investigations"); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + jsonGenerator.writeObjectFieldStart("creators"); + jsonGenerator.writeArrayFieldStart("data"); + for(int personID : relationships.getCreators()) { + jsonGenerator.writeStartObject(); + jsonGenerator.writeNumberField("id", personID); + jsonGenerator.writeStringField("type", "people"); + jsonGenerator.writeEndObject(); + } + jsonGenerator.writeEndArray(); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + } + } + + private class Attributes { + + private String title; + private String description = ""; + private String experimentalists = ""; + private String otherCreators = ""; + + public Attributes(String title) { + this.title = title; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public String getExperimentalists() { + return experimentalists; + } + + public String getOther_creators() { + return otherCreators; + } + } + +} From c41c6aa529f36351cda04a2471ee6b41f451c3f7 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 7 Oct 2024 11:07:51 +0200 Subject: [PATCH 2/4] structure --- .../qbic/model/{ => isa}/SeekStructure.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) rename src/main/java/life/qbic/model/{ => isa}/SeekStructure.java (51%) diff --git a/src/main/java/life/qbic/model/SeekStructure.java b/src/main/java/life/qbic/model/isa/SeekStructure.java similarity index 51% rename from src/main/java/life/qbic/model/SeekStructure.java rename to src/main/java/life/qbic/model/isa/SeekStructure.java index 5d7cc46..323876c 100644 --- a/src/main/java/life/qbic/model/SeekStructure.java +++ b/src/main/java/life/qbic/model/isa/SeekStructure.java @@ -3,18 +3,22 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.util.List; import java.util.Map; +import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.ISAAssay; -import life.qbic.model.isa.ISADataFile; import life.qbic.model.isa.ISASample; public class SeekStructure { private ISAAssay assay; private List samples; - private List dataFiles; - private Map isaToOpenBISFile; - private Map isaFileToAssetType; - //ISASample isaSample = new ISASample(map, sampleTypeID, Arrays.asList(projectID)); + private Map isaToOpenBISFile; + + public SeekStructure(ISAAssay assay, List samples, + Map isaToOpenBISFile) { + this.assay = assay; + this.samples = samples; + this.isaToOpenBISFile = isaToOpenBISFile; + } public ISAAssay getAssay() { return assay; @@ -24,11 +28,8 @@ public List getSamples() { return samples; } - public Map getIsaToOpenBISFiles() { + public Map getISAFileToDatasetFiles() { return isaToOpenBISFile; } - public String getAssetType(ISADataFile dataFile) { - return isaFileToAssetType.get(dataFile); - } } From 3ec62fd4b7068cd7c58ac3c9cec8a4da6650b11c Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 7 Oct 2024 12:50:54 +0200 Subject: [PATCH 3/4] implement first seek use cases --- .../io/commandline/CommandLineOptions.java | 4 +- .../TransferDataToSeekCommand.java | 103 ++++---- .../TransferSampleTypesToSeekCommand.java | 61 +++++ .../qbic/model/AssayWithQueuedAssets.java | 23 ++ .../OpenbisExperimentWithDescendants.java | 4 +- .../qbic/model/OpenbisSeekTranslator.java | 175 ++++++++++---- .../qbic/model/SampleTypesAndMaterials.java | 23 ++ .../qbic/model/download/OpenbisConnector.java | 40 +++- .../qbic/model/download/SEEKConnector.java | 155 +++++++++--- .../life/qbic/model/isa/GenericSeekAsset.java | 199 ++++++++++++++++ .../java/life/qbic/model/isa/ISAAssay.java | 9 - .../java/life/qbic/model/isa/ISADataFile.java | 5 - .../java/life/qbic/model/isa/ISASample.java | 32 +-- .../life/qbic/model/isa/ISASampleType.java | 220 ++++++++++++++++++ .../java/life/qbic/model/isa/ISAStudy.java | 6 - .../life/qbic/model/isa/SeekStructure.java | 5 +- 16 files changed, 899 insertions(+), 165 deletions(-) create mode 100644 src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java create mode 100644 src/main/java/life/qbic/model/AssayWithQueuedAssets.java create mode 100644 src/main/java/life/qbic/model/SampleTypesAndMaterials.java create mode 100644 src/main/java/life/qbic/model/isa/GenericSeekAsset.java create mode 100644 src/main/java/life/qbic/model/isa/ISASampleType.java diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index a0dcbad..b583d9c 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -8,9 +8,9 @@ // main command with format specifiers for the usage help message @Command(name = "openbis-scripts", - subcommands = {SampleHierarchyCommand.class, FindDatasetsCommand.class, + subcommands = {SampleHierarchyCommand.class, TransferSampleTypesToSeekCommand.class, DownloadPetabCommand.class, UploadPetabResultCommand.class, UploadDatasetCommand.class, - SpaceStatisticsCommand.class, TransferDataToSeekCommand.class}, + SpaceStatisticsCommand.class, TransferDataToSeekCommand.class, FindDatasetsCommand.class}, description = "A client software for querying openBIS.", mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) public class CommandLineOptions { diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index ec9bdaf..72d8934 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -9,15 +9,19 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import life.qbic.App; +import life.qbic.model.AssayWithQueuedAssets; import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.OpenbisSeekTranslator; -import life.qbic.model.SeekStructure; +import life.qbic.model.isa.SeekStructure; import life.qbic.model.download.OpenbisConnector; import life.qbic.model.download.SEEKConnector; import life.qbic.model.download.SEEKConnector.AssetToUpload; @@ -65,21 +69,21 @@ public class TransferDataToSeekCommand implements Runnable { AuthenticationOptions auth = new AuthenticationOptions(); OpenbisConnector openbis; SEEKConnector seek; - OpenbisSeekTranslator translator = new OpenbisSeekTranslator(); + OpenbisSeekTranslator translator; @Override public void run() { - System.out.println("auth..."); + System.out.println("Connecting to openBIS..."); OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); - System.out.println("openbis..."); openbis = new OpenbisConnector(authentication); boolean isSample = false; boolean isDataSet = false; - System.out.println("search for experiment..."); + + System.out.println("Searching for specified object in openBIS..."); boolean isExperiment = experimentExists(objectID); if (!isExperiment && sampleExists(objectID)) { @@ -96,32 +100,15 @@ public void run() { objectID); return; } - System.out.println("Searching done..."); -/* - if (isExperiment) { - if (seekNode != null && !seekNode.contains("assays")) { - System.out.printf( - "Seek node %s does not correspond to the provided openBIS experiment identifier. " - + "Please select an assay node.%n", objectID); - return; - } - } - if (isSample) { - if (seekNode != null && !seekNode.contains("samples")) { - System.out.printf( - "Seek node %s does not correspond to the provided openBIS sample identifier. " - + "Please select a sample node.%n", objectID); - return; - } - } - - */ + System.out.println("Search successful."); + System.out.println("Connecting to SEEK..."); byte[] httpCredentials = Base64.encodeBase64( (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); try { - seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "Default Project", + seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "seek_test", "lisym default study"); + translator = seek.getTranslator(); } catch (URISyntaxException | IOException | InterruptedException e) { throw new RuntimeException(e); } @@ -147,13 +134,22 @@ public void run() { */ try { + System.out.println("Collecting information from openBIS..."); OpenbisExperimentWithDescendants experiment = openbis.getExperimentWithDescendants(objectID); - String assayID = getAssayIDForOpenBISExperiment(experiment.getExperiment()); - List blacklist = parseBlackList(blacklistFile); - if(assayID.isBlank()) { - createNewNodes(experiment, blacklist); + System.out.println("Trying to find existing corresponding assay in SEEK..."); + Optional assayID = getAssayIDForOpenBISExperiment(experiment.getExperiment()); + assayID.ifPresent(x -> System.out.println("Found assay with id "+assayID.get())); + Set blacklist = parseBlackList(blacklistFile); + System.out.println("Translating openBIS property codes to SEEK names..."); + Map sampleTypesToIds = seek.getSampleTypeNamesToIDs(); + System.out.println("Creating SEEK structure..."); + SeekStructure nodeWithChildren = translator.translate(experiment, sampleTypesToIds, blacklist); + if(assayID.isEmpty()) { + System.out.println("Creating new nodes..."); + createNewNodes(nodeWithChildren); } else { - updateNodes(experiment, assayID, blacklist); + System.out.println("Updating nodes..."); + updateNodes(nodeWithChildren, assayID.get()); } } catch (URISyntaxException | InterruptedException | IOException e) { throw new RuntimeException(e); @@ -187,42 +183,46 @@ public void run() { System.out.println("Done"); } - private List parseBlackList(String blacklistFile) { - List result = new ArrayList<>(); + private Set parseBlackList(String blacklistFile) { if(blacklistFile == null) { - return result; + return new HashSet<>(); } // trim whitespace, skip empty lines try (Stream lines = Files.lines(Paths.get(blacklistFile)) .map(String::trim) .filter(s -> !s.isBlank())) { - return lines.collect(Collectors.toList()); + + Set codes = lines.collect(Collectors.toSet()); + + for(String code : codes) { + if(!OpenbisConnector.datasetCodePattern.matcher(code).matches()) { + throw new RuntimeException("Invalid dataset code: " + code+". Please make sure to use valid" + + " dataset codes in the blacklist file."); + } + } + return codes; } catch (IOException e) { throw new RuntimeException(blacklistFile+" could not be found or read."); } } - private void updateNodes(OpenbisExperimentWithDescendants experiment, String assayID, List blacklist) { - System.err.println("updating nodes of assay "+assayID); - SeekStructure nodeWithChildren = translator.translate(experiment, blacklist); + private void updateNodes(SeekStructure nodeWithChildren, String assayID) { String updatedEndpoint = seek.updateNode(nodeWithChildren, assayID, transferData); - System.out.printf("%s was successfully updated.%n", endpoint); + System.out.printf("%s was successfully updated.%n", updatedEndpoint); } - private void createNewNodes(OpenbisExperimentWithDescendants experiment, List blacklist) + private void createNewNodes(SeekStructure nodeWithChildren) throws URISyntaxException, IOException, InterruptedException { - System.err.println("creating new nodes"); - SeekStructure nodeWithChildren = translator.translate(experiment, blacklist); - List assetsToUpload = seek.createNode(nodeWithChildren, transferData); + AssayWithQueuedAssets assetsOfAssayToUpload = seek.createNode(nodeWithChildren, transferData); if(transferData) { - for(AssetToUpload asset : assetsToUpload) { + for(AssetToUpload asset : assetsOfAssayToUpload.getAssets()) { System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), () -> openbis.streamDataset(asset.getDataSetCode(), asset.getFilePath())); System.out.printf("File stored here: %s%n", fileURL); } } - System.out.printf("%s was successfully created.%n", endpoint); + System.out.printf("%s was successfully created.%n", assetsOfAssayToUpload.getAssayEndpoint()); } private boolean sampleExists(String objectID) { @@ -237,7 +237,7 @@ private boolean experimentExists(String experimentID) { return openbis.experimentExists(experimentID); } - private String getAssayIDForOpenBISExperiment(Experiment experiment) + private Optional getAssayIDForOpenBISExperiment(Experiment experiment) throws URISyntaxException, IOException, InterruptedException { // the perm id is unique and afaik not used by scientists. it is highly unlikely that it would // "accidentally" be part of another title or description. however, care should be taken here, @@ -246,12 +246,12 @@ private String getAssayIDForOpenBISExperiment(Experiment experiment) String permID = experiment.getPermId().getPermId(); List assayIDs = seek.searchAssaysContainingKeyword(permID); if(assayIDs.isEmpty()) { - return ""; + return Optional.empty(); } if(assayIDs.size() == 1) { - return assayIDs.get(0); + return Optional.of(assayIDs.get(0)); } - throw new RuntimeException("Search term "+permID+ " was found in more than one assay: "+assayIDs); + throw new RuntimeException("Experiment identifier "+permID+ " was found in more than one assay: "+assayIDs); } private void sendDatasetToSeek(String datasetCode, String assayID) @@ -265,8 +265,7 @@ private void sendDatasetToSeek(String datasetCode, String assayID) } DataSet dataset = datasets.get(0); List files = openbis.getDatasetFiles(dataset); - List assets = seek.createAssets(files, dataset.getType().getCode(), - Arrays.asList(assayID)); + List assets = seek.createAssets(files, Arrays.asList(assayID)); for(AssetToUpload asset : assets) { System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), diff --git a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java new file mode 100644 index 0000000..cdd6ee4 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java @@ -0,0 +1,61 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import java.io.IOException; +import java.net.URISyntaxException; +import life.qbic.App; +import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.SampleTypesAndMaterials; +import life.qbic.model.download.OpenbisConnector; +import life.qbic.model.download.SEEKConnector; +import org.apache.commons.codec.binary.Base64; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; + +@Command(name = "sample-type-transfer", + description = + "Transfers sample types from openBIS to SEEK.") +public class TransferSampleTypesToSeekCommand implements Runnable { + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisConnector openbis; + SEEKConnector seek; + OpenbisSeekTranslator translator; + + @Override + public void run() { + System.out.println("auth..."); + + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getAS(), auth.getDSS()); + System.out.println("openbis..."); + + openbis = new OpenbisConnector(authentication); + + byte[] httpCredentials = Base64.encodeBase64( + (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); + try { + seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "seek_test", + "lisym default study"); + translator = seek.getTranslator(); + } catch (URISyntaxException | IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + SampleTypesAndMaterials types = openbis.getSampleTypesWithMaterials(); + + try { + for(SampleType type : types.getSampleTypes()) { + System.err.println("creating "+type.getCode()); + String sampleTypeId = seek.createSampleType(translator.translate(type)); + System.err.println("created "+sampleTypeId); + } + } catch (URISyntaxException | IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + System.out.println("Done"); + } + +} diff --git a/src/main/java/life/qbic/model/AssayWithQueuedAssets.java b/src/main/java/life/qbic/model/AssayWithQueuedAssets.java new file mode 100644 index 0000000..0186cfc --- /dev/null +++ b/src/main/java/life/qbic/model/AssayWithQueuedAssets.java @@ -0,0 +1,23 @@ +package life.qbic.model; + +import java.util.List; +import life.qbic.model.download.SEEKConnector.AssetToUpload; + +public class AssayWithQueuedAssets { + + private String assayEndpoint; + private List assetsToUpload; + + public AssayWithQueuedAssets(String assayEndpoint, List assetsToUpload) { + this.assayEndpoint = assayEndpoint; + this.assetsToUpload = assetsToUpload; + } + + public String getAssayEndpoint() { + return assayEndpoint; + } + + public List getAssets() { + return assetsToUpload; + } +} diff --git a/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java index 86399ec..0782d55 100644 --- a/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java +++ b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java @@ -12,10 +12,10 @@ public class OpenbisExperimentWithDescendants { private Experiment experiment; private List samples; private List datasets; - private Map> datasetCodeToFiles; + private Map> datasetCodeToFiles; public OpenbisExperimentWithDescendants(Experiment experiment, List samples, - List datasets, Map> datasetCodeToFiles) { + List datasets, Map> datasetCodeToFiles) { this.experiment = experiment; this.samples = samples; this.datasets = datasets; diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index a02317d..0504636 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -4,20 +4,36 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.DataType; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.PropertyAssignment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; +import life.qbic.model.isa.GenericSeekAsset; +import life.qbic.model.isa.ISAAssay; import life.qbic.model.isa.ISADataFile; import life.qbic.model.isa.ISASample; +import life.qbic.model.isa.ISASampleType; +import life.qbic.model.isa.ISASampleType.SampleAttribute; +import life.qbic.model.isa.ISASampleType.SampleAttributeType; +import life.qbic.model.isa.SeekStructure; public class OpenbisSeekTranslator { private final String DEFAULT_PROJECT_ID; private final String DEFAULT_STUDY_ID; + private final String DEFAULT_TRANSFERRED_SAMPLE_TITLE = "openBIS Name"; public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { this.DEFAULT_PROJECT_ID = defaultProjectID; @@ -34,9 +50,27 @@ public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { entry("04_MICRO_CT", "EXP") ); + Map dataTypeToAttributeType = Map.ofEntries( + entry(DataType.INTEGER, new SampleAttributeType("4", "Integer", "Integer")), + entry(DataType.VARCHAR, new SampleAttributeType("8", "String", "String")), + entry(DataType.MULTILINE_VARCHAR, new SampleAttributeType("7", "Text", "Text")), + entry(DataType.REAL, new SampleAttributeType("3", "Real number", "Float")), + entry(DataType.TIMESTAMP, new SampleAttributeType("1", "Date time", "DateTime")), + entry(DataType.BOOLEAN, new SampleAttributeType("16", "Boolean", "Boolean")), + entry(DataType.CONTROLLEDVOCABULARY, //we use String for now + new SampleAttributeType("8", "String", "String")), + entry(DataType.MATERIAL, //not used anymore in this form + new SampleAttributeType("8", "String", "String")), + entry(DataType.HYPERLINK, new SampleAttributeType("8", "String", "String")), + entry(DataType.XML, new SampleAttributeType("7", "Text", "Text")), + entry(DataType.SAMPLE, //should be handled before mapping types + new SampleAttributeType("8", "String", "String")), + entry(DataType.DATE, new SampleAttributeType("2", "Date time", "Date")) + ); + Map experimentTypeToAssayType = Map.ofEntries( entry("00_MOUSE_DATABASE", ""), - entry("00_PATIENT_DATABASE", ""),//if this is related to measured data, attach it as sample to the assay + entry("00_PATIENT_DATABASE", ""), entry("00_STANDARD_OPERATING_PROTOCOLS", ""), entry("01_BIOLOGICAL_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Cultivation_experiment"), entry("02_MASSSPECTROMETRY_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Proteomics"), @@ -55,67 +89,128 @@ public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { ); Map datasetTypeToAssetType = Map.ofEntries( - entry("ANALYSIS_NOTEBOOK", "Document"), - entry("ANALYZED_DATA", "Data_file"), - entry("ATTACHMENT", "Document"), + entry("ANALYSIS_NOTEBOOK", "documents"), + entry("ANALYZED_DATA", "data_files"), + entry("ATTACHMENT", "documents"), entry("ELN_PREVIEW", ""), - entry("EXPERIMENT_PROTOCOL", "SOP"), - entry("EXPERIMENT_RESULT", "Document"), - entry("HISTOLOGICAL_SLIDE", "Data_file"), - entry("IB_DATA", "Data_file"), - entry("LUMINEX_DATA", "Data_file"), - entry("MS_DATA_ANALYZED", "Data_file"), - entry("MS_DATA_RAW", "Data_file"), - entry("OTHER_DATA", "Document"), - entry("PROCESSED_DATA", "Data_file"), - entry("PUBLICATION_DATA", "Publication"), - entry("QPCR_DATA", "Data_file"), - entry("RAW_DATA", "Data_file"), - entry("SOURCE_CODE", "Document"), + entry("EXPERIMENT_PROTOCOL", "sops"), + entry("EXPERIMENT_RESULT", "documents"), + entry("HISTOLOGICAL_SLIDE", "data_files"), + entry("IB_DATA", "data_files"), + entry("LUMINEX_DATA", "data_files"), + entry("MS_DATA_ANALYZED", "data_files"), + entry("MS_DATA_RAW", "data_files"), + entry("OTHER_DATA", "data_files"), + entry("PROCESSED_DATA", "data_files"), + entry("PUBLICATION_DATA", "publications"), + entry("QPCR_DATA", "data_files"), + entry("RAW_DATA", "data_files"), + entry("SOURCE_CODE", "documents"), entry("TEST_CONT", ""), entry("TEST_DAT", ""), - entry("UNKNOWN", "") + entry("UNKNOWN", "data_files") ); - public SeekStructure translate(String seekNode, Experiment experimentWithSamplesAndDatasets) { - Experiment exp = experimentWithSamplesAndDatasets; - exp.getType(); - return null; - //new ISAAssay(exp.getCode(), ) - } - - public ISADataFile translate(String seekNode, DataSet dataset) { - return null; //new ISADataFile(); + public ISASampleType translate(SampleType sampleType) { + SampleAttribute titleAttribute = new SampleAttribute(DEFAULT_TRANSFERRED_SAMPLE_TITLE, + dataTypeToAttributeType.get(DataType.VARCHAR), true, false); + ISASampleType type = new ISASampleType(sampleType.getCode(), titleAttribute, + DEFAULT_PROJECT_ID); + for (PropertyAssignment a : sampleType.getPropertyAssignments()) { + DataType dataType = a.getPropertyType().getDataType(); + type.addSampleAttribute(a.getPropertyType().getLabel(), dataTypeToAttributeType.get(dataType), + false, null); + } + return type; } public String assetForDatasetType(String datasetType) { - return "Data_file";//TODO + if(datasetTypeToAssetType.get(datasetType) == null || datasetTypeToAssetType.get(datasetType).isBlank()) { + throw new RuntimeException("Dataset type " + datasetType + " could not be mapped to SEEK type."); + } + return datasetTypeToAssetType.get(datasetType); } public String dataFormatAnnotationForExtension(String fileExtension) { return fileExtensionToDataFormat.get(fileExtension); } - public SeekStructure translate(Experiment experiment, List blacklist) { - System.err.println(experiment.getCode()); + public SeekStructure translate(OpenbisExperimentWithDescendants experiment, + Map sampleTypesToIds, Set blacklist) throws URISyntaxException { + + Experiment exp = experiment.getExperiment(); + String expType = exp.getType().getCode(); + String title = exp.getCode()+" ("+exp.getPermId().getPermId()+")"; + ISAAssay assay = new ISAAssay(title, DEFAULT_STUDY_ID, experimentTypeToAssayClass.get(expType), + new URI(experimentTypeToAssayType.get(expType)));//TODO + List samples = new ArrayList<>(); for(Sample sample : experiment.getSamples()) { - String sampleType = getSampleType(sample.getType()); + SampleType sampleType = sample.getType(); + + //try to put all attributes into sample properties, as they should be a 1:1 mapping + Map typeCodesToNames = new HashMap<>(); + for (PropertyAssignment a : sampleType.getPropertyAssignments()) { + typeCodesToNames.put(a.getPropertyType().getCode(), a.getPropertyType().getLabel()); + } Map attributes = new HashMap<>(); - ISASample isaSample = new ISASample(attributes, sampleType, + for(String code : sample.getProperties().keySet()) { + attributes.put(typeCodesToNames.get(code), sample.getProperties().get(code)); + } + + attributes.put(DEFAULT_TRANSFERRED_SAMPLE_TITLE, sample.getIdentifier().getIdentifier()); + String sampleTypeId = sampleTypesToIds.get(sampleType.getCode()); + ISASample isaSample = new ISASample(sample.getPermId().getPermId(), attributes, sampleTypeId, Collections.singletonList(DEFAULT_PROJECT_ID)); - System.err.println(sample.getCode()); + samples.add(isaSample); } - for(DataSet dataset : experiment.getDataSets()) { - System.err.println(dataset.getCode()); + Map isaToOpenBISFile = new HashMap<>(); + + //create ISA files for assets. If actual data is uploaded is determined later based upon flag + for(DatasetWithProperties dataset : experiment.getDatasets()) { + String permID = dataset.getCode(); + if(!blacklist.contains(permID)) { + for(DataSetFile file : experiment.getFilesForDataset(permID)) { + String datasetType = getDatasetTypeOfFile(file, experiment.getDatasets()); + datasetFileToSeekAsset(file, datasetType) + .ifPresent(seekAsset -> isaToOpenBISFile.put(seekAsset, file)); + } + } } - System.err.println("blacklisted:"); - for(String dataset : blacklist) { - System.err.println(dataset); + return new SeekStructure(assay, samples, isaToOpenBISFile); + } + + private String getDatasetTypeOfFile(DataSetFile file, List dataSets) { + String permId = file.getDataSetPermId().getPermId(); + for(DatasetWithProperties dataset : dataSets) { + if(dataset.getCode().equals(permId)) { + return dataset.getType().getCode(); + } } - return null;//TODO + return ""; } - private String getSampleType(SampleType type) { + /** + * Creates a SEEK asset from an openBIS DataSetFile, if it describes a file (not a folder). + * @param file the openBIS DataSetFile + * @return an optional SEEK asset + */ + private Optional datasetFileToSeekAsset(DataSetFile file, String datasetType) { + if (!file.getPath().isBlank() && !file.isDirectory()) { + File f = new File(file.getPath()); + String datasetCode = file.getDataSetPermId().toString(); + String assetName = datasetCode + ": " + f.getName(); + String assetType = assetForDatasetType(datasetType); + GenericSeekAsset isaFile = new GenericSeekAsset(assetType, assetName, file.getPath(), + Arrays.asList(DEFAULT_PROJECT_ID)); + String fileExtension = f.getName().substring(f.getName().lastIndexOf(".") + 1); + String annotation = dataFormatAnnotationForExtension(fileExtension); + if (annotation != null) { + isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); + } + return Optional.of(isaFile); + } + return Optional.empty(); } + } diff --git a/src/main/java/life/qbic/model/SampleTypesAndMaterials.java b/src/main/java/life/qbic/model/SampleTypesAndMaterials.java new file mode 100644 index 0000000..8a82576 --- /dev/null +++ b/src/main/java/life/qbic/model/SampleTypesAndMaterials.java @@ -0,0 +1,23 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import java.util.Set; + +public class SampleTypesAndMaterials { + + Set sampleTypes; + Set sampleTypesAsMaterials; + + public SampleTypesAndMaterials(Set sampleTypes, Set sampleTypesAsMaterials) { + this.sampleTypes = sampleTypes; + this.sampleTypesAsMaterials = sampleTypesAsMaterials; + } + + public Set getSamplesAsMaterials() { + return sampleTypesAsMaterials; + } + + public Set getSampleTypes() { + return sampleTypes; + } +} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index e50dd12..7e5d7ed 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -12,11 +12,15 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.search.ExperimentSearchCriteria; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.PropertyAssignment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleTypeFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SamplePermId; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleTypeSearchCriteria; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria; @@ -37,14 +41,18 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; +import life.qbic.App; import life.qbic.model.DatasetWithProperties; import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.SampleTypeConnection; +import life.qbic.model.SampleTypesAndMaterials; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -428,15 +436,16 @@ public OpenbisExperimentWithDescendants getExperimentWithDescendants(String expe dataSetFetchOptions.withRegistrator(); SampleFetchOptions sampleFetchOptions = new SampleFetchOptions(); sampleFetchOptions.withProperties(); + sampleFetchOptions.withType().withPropertyAssignments().withPropertyType(); sampleFetchOptions.withDataSetsUsing(dataSetFetchOptions); fetchOptions.withDataSetsUsing(dataSetFetchOptions); fetchOptions.withSamplesUsing(sampleFetchOptions); Experiment experiment = openBIS.searchExperiments(criteria, fetchOptions).getObjects().get(0); - Map> datasetCodeToFiles = new HashMap<>(); + Map> datasetCodeToFiles = new HashMap<>(); for(DataSet dataset : experiment.getDataSets()) { - datasetCodeToFiles.put(dataset.getPermId(), getDatasetFiles(dataset)); + datasetCodeToFiles.put(dataset.getPermId().getPermId(), getDatasetFiles(dataset)); } return new OpenbisExperimentWithDescendants(experiment, experiment.getSamples(), @@ -455,4 +464,31 @@ public List getDatasetFiles(DataSet dataset) { return result.getObjects(); } + + public SampleTypesAndMaterials getSampleTypesWithMaterials() { + SampleTypeSearchCriteria criteria = new SampleTypeSearchCriteria(); + SampleTypeFetchOptions typeOptions = new SampleTypeFetchOptions(); + typeOptions.withPropertyAssignments().withPropertyType(); + typeOptions.withPropertyAssignments().withEntityType(); + Set sampleTypes = new HashSet<>(); + Set sampleTypesAsMaterials = new HashSet<>(); + for(SampleType type : openBIS.searchSampleTypes(criteria, typeOptions).getObjects()) { + /* + System.err.println("sample type: "+type.getCode()); + for(PropertyAssignment assignment : type.getPropertyAssignments()) { + if (assignment.getPropertyType().getDataType().name().equals("SAMPLE")) { + System.err.println(assignment.getPropertyType().getLabel()); + System.err.println(assignment.getPropertyType().getDataType().name()); + System.err.println(assignment.getPropertyType().getCode()); + } + } + */ + if(type.getCode().startsWith("MATERIAL.")) { + sampleTypesAsMaterials.add(type); + } else { + sampleTypes.add(type); + } + } + return new SampleTypesAndMaterials(sampleTypes, sampleTypesAsMaterials); + } } diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java index 298364e..580bf0f 100644 --- a/src/main/java/life/qbic/model/download/SEEKConnector.java +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -1,6 +1,7 @@ package life.qbic.model.download; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; @@ -21,12 +22,15 @@ import java.util.Map; import java.util.Optional; import java.util.function.Supplier; +import life.qbic.model.AssayWithQueuedAssets; import life.qbic.model.OpenbisSeekTranslator; -import life.qbic.model.SeekStructure; +import life.qbic.model.isa.SeekStructure; +import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.ISAAssay; -import life.qbic.model.isa.ISADataFile; import life.qbic.model.isa.ISASample; +import life.qbic.model.isa.ISASampleType; import life.qbic.model.isa.ISAStudy; +import org.apache.commons.codec.binary.Base64; import org.apache.http.client.utils.URIBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -42,8 +46,8 @@ public SEEKConnector(String apiURL, byte[] httpCredentials, String defaultProjec String defaultStudyTitle) throws URISyntaxException, IOException, InterruptedException { this.apiURL = apiURL; this.credentials = httpCredentials; - translator = new OpenbisSeekTranslator(searchNodeWithTitle("projects", defaultProjectTitle), - searchNodeWithTitle("studies", defaultStudyTitle)); + translator = new OpenbisSeekTranslator("1", //searchNodeWithTitle("projects", defaultProjectTitle), + "1");//searchNodeWithTitle("studies", defaultStudyTitle)); } public String addAssay(ISAAssay assay) @@ -55,7 +59,6 @@ public String addAssay(ISAAssay assay) BodyHandlers.ofString()); if(response.statusCode()!=200) { - System.err.println(response.body()); throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } JsonNode rootNode = new ObjectMapper().readTree(response.body()); @@ -103,6 +106,18 @@ public boolean studyExists(String id) throws URISyntaxException, IOException, In return response.statusCode() == 200; } + public void printAttributeTypes() throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/sample_attribute_types"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + System.err.println(response.body()); + } /* -datatype by extension -assay equals experiment @@ -112,18 +127,52 @@ public boolean studyExists(String id) throws URISyntaxException, IOException, In -flexible object type to sample type? */ + public void deleteSampleType(String id) throws URISyntaxException, IOException, + InterruptedException { + String endpoint = apiURL+"/sample_types"; + URIBuilder builder = new URIBuilder(endpoint); + builder.setParameter("id", id); + + HttpResponse response = HttpClient.newBuilder().build() + .send(HttpRequest.newBuilder().uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .DELETE().build(), BodyHandlers.ofString()); + + if(response.statusCode()!=201) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + public String createSampleType(ISASampleType sampleType) throws URISyntaxException, IOException, + InterruptedException { + String endpoint = apiURL+"/sample_types"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, sampleType.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=201) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + public String createSample(ISASample isaSample) throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/samples"; - HashMap map = new HashMap<>(); - //map.put("title", "really?"); - map.put("test_attribute", "okay then"); HttpResponse response = HttpClient.newBuilder().build() .send(buildAuthorizedPOSTRequest(endpoint, isaSample.toJson()), BodyHandlers.ofString()); - if(response.statusCode()!=201) { + if(response.statusCode()!=200) { System.err.println(response.body()); throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } @@ -133,9 +182,9 @@ public String createSample(ISASample isaSample) throws URISyntaxException, IOExc return idNode.asText(); } - private AssetToUpload createDataFileAsset(String datasetCode, ISADataFile data) + private AssetToUpload createDataFileAsset(String datasetCode, GenericSeekAsset data) throws IOException, URISyntaxException, InterruptedException { - String endpoint = apiURL+"/data_files"; + String endpoint = apiURL+"/"+data.getType(); HttpResponse response = HttpClient.newBuilder().build() .send(buildAuthorizedPOSTRequest(endpoint, data.toJson()), @@ -228,8 +277,8 @@ public Optional createAssetForFile(DataSetFile file, String asset File f = new File(file.getPath()); String datasetCode = file.getDataSetPermId().toString(); String assetName = datasetCode+": "+f.getName(); - ISADataFile isaFile = new ISADataFile(assetName, file.getPath(), - Arrays.asList("1")); + GenericSeekAsset isaFile = new GenericSeekAsset("data_files", assetName, file.getPath(), + Arrays.asList("1"));//TODO isaFile.withAssays(assays); String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); String annotation = translator.dataFormatAnnotationForExtension(fileExtension); @@ -241,7 +290,7 @@ public Optional createAssetForFile(DataSetFile file, String asset return Optional.empty(); } - public List createAssets(List filesInDataset, String assetType, + public List createAssets(List filesInDataset, List assays) throws IOException, URISyntaxException, InterruptedException { List result = new ArrayList<>(); @@ -249,8 +298,8 @@ public List createAssets(List filesInDataset, String if(!file.getPath().isBlank() && !file.isDirectory()) { File f = new File(file.getPath()); String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode+": "+f.getName(); - ISADataFile isaFile = new ISADataFile(assetName, file.getPath(), + String assetName = datasetCode+": "+f.getName();//TODO? + GenericSeekAsset isaFile = new GenericSeekAsset("data_files", assetName, file.getPath(), Arrays.asList("1")); isaFile.withAssays(assays); String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); @@ -264,6 +313,27 @@ public List createAssets(List filesInDataset, String return result; } + /** + * Creates + * @param isaToOpenBISFile + * @param assays + * @return + * @throws IOException + * @throws URISyntaxException + * @throws InterruptedException + */ + public List createAssetsForAssays(Map isaToOpenBISFile, + List assays) + throws IOException, URISyntaxException, InterruptedException { + List result = new ArrayList<>(); + for (GenericSeekAsset isaFile : isaToOpenBISFile.keySet()) { + isaFile.withAssays(assays); + result.add(createDataFileAsset(isaToOpenBISFile.get(isaFile).getDataSetPermId().getPermId(), + isaFile)); + } + return result; + } + public String listAssays() throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/assays/"; HttpRequest request = HttpRequest.newBuilder() @@ -281,11 +351,42 @@ public String listAssays() throws URISyntaxException, IOException, InterruptedEx } } + public Map getSampleTypeNamesToIDs() + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/sample_types/"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + return parseSampleTypesJSON(response.body()); + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + private Map parseSampleTypesJSON(String json) throws JsonProcessingException { + Map typesToIDs = new HashMap<>(); + JsonNode rootNode = new ObjectMapper().readTree(json); + JsonNode hits = rootNode.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + String id = hit.get("id").asText(); + String title = hit.get("attributes").get("title").asText(); + typesToIDs.put(title, id); + } + return typesToIDs; + } + private String searchNodeWithTitle(String nodeType, String title) throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/search/"; URIBuilder builder = new URIBuilder(endpoint); - builder.setParameter("q", title).setParameter("type", "nodeType"); + builder.setParameter("q", title).setParameter("search_type", nodeType); HttpRequest request = HttpRequest.newBuilder() .uri(builder.build()) @@ -295,11 +396,13 @@ private String searchNodeWithTitle(String nodeType, String title) .GET().build(); HttpResponse response = HttpClient.newBuilder().build() .send(request, BodyHandlers.ofString()); + System.err.println("searching for: "+title+" ("+nodeType+")"); if(response.statusCode() == 200) { JsonNode rootNode = new ObjectMapper().readTree(response.body()); JsonNode hits = rootNode.path("data"); for (Iterator it = hits.elements(); it.hasNext(); ) { JsonNode hit = it.next(); + System.err.println(hit.asText()); if(hit.get("title").asText().equals(title)) { return hit.get("id").asText(); } @@ -348,26 +451,24 @@ public List searchAssaysContainingKeyword(String searchTerm) public String updateNode(SeekStructure nodeWithChildren, String assayID, boolean transferData) { //updateAssay(nodeWithChildren.getAssay()); - return assayID; + return apiURL+"/assays/"+assayID; } - public List createNode(SeekStructure nodeWithChildren, boolean transferData) + public AssayWithQueuedAssets createNode(SeekStructure nodeWithChildren, boolean transferData) throws URISyntaxException, IOException, InterruptedException { String assayID = addAssay(nodeWithChildren.getAssay()); for(ISASample sample : nodeWithChildren.getSamples()) { createSample(sample); } - List assets = new ArrayList<>(); + Map isaToFileMap = nodeWithChildren.getISAFileToDatasetFiles(); - Map isaToFileMap = nodeWithChildren.getIsaToOpenBISFiles(); - for(ISADataFile isaFile : isaToFileMap.keySet()) { - String assetType = nodeWithChildren.getAssetType(isaFile); - createAssetForFile(isaToFileMap.get(isaFile), assetType, Arrays.asList(assayID)) - .ifPresent(assets::add); - } + return new AssayWithQueuedAssets(apiURL+"/assays/"+assayID, + createAssetsForAssays(isaToFileMap, Arrays.asList(assayID))); + } - return assets; + public OpenbisSeekTranslator getTranslator() { + return translator; } public static class AssetToUpload { diff --git a/src/main/java/life/qbic/model/isa/GenericSeekAsset.java b/src/main/java/life/qbic/model/isa/GenericSeekAsset.java new file mode 100644 index 0000000..34b4fef --- /dev/null +++ b/src/main/java/life/qbic/model/isa/GenericSeekAsset.java @@ -0,0 +1,199 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Model class for Seek assets. Contains all mandatory and some optional properties and attributes + * that are needed to create an asset in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. Since there are different types of assets, the isaType is a + * parameter here. + */ +public class GenericSeekAsset extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private String assetType; + + public GenericSeekAsset(String assetType, String title, String fileName, List projectIds) { + this.assetType = assetType; + this.attributes = new Attributes(title, fileName); + this.relationships = new Relationships(projectIds); + } + + public GenericSeekAsset withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public GenericSeekAsset withAssays(List assays) { + this.relationships.assays = assays; + return this; + } + + public GenericSeekAsset withDataFormatAnnotations(List identifiers) { + this.attributes.withDataFormatAnnotations(identifiers); + return this; + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return assetType; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public String getFileName() { + return attributes.getContent_blobs().get(0).getOriginal_filename(); + } + + private class Relationships { + + private List projects; + private List assays; + + public Relationships(List projects) { + this.projects = projects; + this.assays = new ArrayList<>(); + } + + public List getProjects() { + return projects; + } + + public List getAssays() { + return assays; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + + generateListJSON(jsonGenerator, "projects", relationships.projects, "projects"); + generateListJSON(jsonGenerator, "assays", relationships.assays, "assays"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, + String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for (String item : items) { + generator.writeStartObject(); + generator.writeStringField("id", item); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + private String title; + private List contentBlobs = new ArrayList<>(); + private String otherCreators = ""; + private List dataFormatAnnotations = new ArrayList<>(); + + + public Attributes(String title, String fileName) { + this.title = title; + this.contentBlobs.add(new ContentBlob(fileName)); + } + + public String getTitle() { + return title; + } + + public List getContent_blobs() { + return contentBlobs; + } + + public String getOther_creators() { + return otherCreators; + } + + public List getData_format_annotations() { + return dataFormatAnnotations; + } + + public void withDataFormatAnnotations(List identifiers) { + List annotations = new ArrayList<>(); + for(String id : identifiers) { + annotations.add(new DataFormatAnnotation(id)); + } + this.dataFormatAnnotations = annotations; + } + + private class DataFormatAnnotation { + + private String label; + private String identifier; + + public DataFormatAnnotation(String identifier) { + this.identifier = identifier; + } + + public String getIdentifier() { + return identifier; + } + + } + + private class ContentBlob { + + private String originalFilename; + private String contentType; + + public ContentBlob(String fileName) { + this.originalFilename = fileName; + String suffix = fileName.substring(fileName.indexOf('.') + 1); + + this.contentType = "application/" + suffix; + } + + public String getContent_type() { + return contentType; + } + + public String getOriginal_filename() { + return originalFilename; + } + } + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISAAssay.java b/src/main/java/life/qbic/model/isa/ISAAssay.java index 188d544..ed2a7c0 100644 --- a/src/main/java/life/qbic/model/isa/ISAAssay.java +++ b/src/main/java/life/qbic/model/isa/ISAAssay.java @@ -88,15 +88,6 @@ public Attributes getAttributes() { return attributes; } - public static void main(String[] args) throws JsonProcessingException, URISyntaxException { - ISAAssay assay = new ISAAssay("title", "1", - "EXP", - new URI("http://jermontology.org/ontology/JERMOntology#RNA-Seq")); - assay.setCreatorIDs(Arrays.asList(3,2)); - assay.setOrganismIDs(Arrays.asList(123,3332)); - System.err.println(assay.toJson()); - } - private class Relationships { private String studyId; diff --git a/src/main/java/life/qbic/model/isa/ISADataFile.java b/src/main/java/life/qbic/model/isa/ISADataFile.java index f2fc346..7c0ddaf 100644 --- a/src/main/java/life/qbic/model/isa/ISADataFile.java +++ b/src/main/java/life/qbic/model/isa/ISADataFile.java @@ -61,11 +61,6 @@ public Attributes getAttributes() { return attributes; } - public static void main(String[] args) throws JsonProcessingException { - ISADataFile sample = new ISADataFile("my file", "myfile.pdf", Arrays.asList("1")); - System.err.println(sample.toJson()); - } - public String getFileName() { return attributes.getContent_blobs().get(0).getOriginal_filename(); } diff --git a/src/main/java/life/qbic/model/isa/ISASample.java b/src/main/java/life/qbic/model/isa/ISASample.java index 5dce2de..1663b99 100644 --- a/src/main/java/life/qbic/model/isa/ISASample.java +++ b/src/main/java/life/qbic/model/isa/ISASample.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -25,9 +24,10 @@ public class ISASample extends AbstractISAObject { private Relationships relationships; private final String ISA_TYPE = "samples"; - public ISASample(Map attributeMap, String sampleType, List projectIds) { - this.attributes = new Attributes(attributeMap); - this.relationships = new Relationships(sampleType, projectIds); + public ISASample(String title, Map attributeMap, String sampleTypeId, + List projectIds) { + this.attributes = new Attributes(title, attributeMap); + this.relationships = new Relationships(sampleTypeId, projectIds); } public ISASample withOtherCreators(String otherCreators) { @@ -61,26 +61,20 @@ public Attributes getAttributes() { return attributes; } - public static void main(String[] args) throws JsonProcessingException { - ISASample sample = new ISASample(new HashMap<>(), "1", Arrays.asList("1")); - sample.setCreatorIDs(Arrays.asList("3","2")); - System.err.println(sample.toJson()); - } - private class Relationships { - private String sampleType; + private String sampleTypeId; private List projects; private List creators = new ArrayList<>(); private List assays = new ArrayList<>(); - public Relationships(String sampleType, List projects) { + public Relationships(String sampleTypeId, List projects) { this.projects = projects; - this.sampleType = sampleType; + this.sampleTypeId = sampleTypeId; } public String getSample_type() { - return sampleType; + return sampleTypeId; } public List getAssays() { @@ -120,7 +114,7 @@ public void serialize(Relationships relationships, JsonGenerator jsonGenerator, jsonGenerator.writeStartObject(); jsonGenerator.writeObjectFieldStart("sample_type"); jsonGenerator.writeObjectFieldStart("data"); - jsonGenerator.writeStringField("id", relationships.sampleType); + jsonGenerator.writeStringField("id", relationships.sampleTypeId); jsonGenerator.writeStringField("type", "sample_types"); jsonGenerator.writeEndObject(); jsonGenerator.writeEndObject(); @@ -151,9 +145,15 @@ private class Attributes { private Map attributeMap = new HashMap<>(); private String otherCreators = ""; + private String title; - public Attributes(Map attributeMap) { + public Attributes(String title, Map attributeMap) { this.attributeMap = attributeMap; + this.title = title; + } + + public String getTitle() { + return title; } public Map getAttribute_map() { diff --git a/src/main/java/life/qbic/model/isa/ISASampleType.java b/src/main/java/life/qbic/model/isa/ISASampleType.java new file mode 100644 index 0000000..201a263 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISASampleType.java @@ -0,0 +1,220 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Model class for SampleType. Contains all mandatory and some optional properties and attributes + * that are needed to create a sample type in SEEK. The model and its getters (names) are structured + * in a way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + * Can be used to populate a SEEK installation with sample types taken from another system's API. + */ +public class ISASampleType extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private final String ISA_TYPE = "sample_types"; + + public ISASampleType(String title, SampleAttribute titleAttribute, String projectID) { + this.attributes = new Attributes(title, titleAttribute); + this.relationships = new Relationships(Arrays.asList(projectID)); + } + + public void addSampleAttribute(String title, SampleAttributeType sampleAttributeType, + boolean required, String linkedSampleTypeIdOrNull) { + attributes.addSampleAttribute(title, sampleAttributeType, required, linkedSampleTypeIdOrNull); + } + + public ISASampleType withAssays(List assays) { + this.relationships.assays = assays; + return this; + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + private class Relationships { + + private List projects; + private List assays; + + public Relationships(List projects) { + this.projects = projects; + this.assays = new ArrayList<>(); + } + + public List getProjects() { + return projects; + } + + public List getAssays() { + return assays; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + + generateListJSON(jsonGenerator, "projects", relationships.projects, "projects"); + generateListJSON(jsonGenerator, "assays", relationships.assays, "assays"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, + String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for (String item : items) { + generator.writeStartObject(); + generator.writeStringField("id", item); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + private String title; + private List sampleAttributes = new ArrayList<>();; + + public Attributes(String title, SampleAttribute titleAttribute) { + this.title = title; + if(!titleAttribute.isTitle) { + throw new IllegalArgumentException("The first sample attribute must be the title attribute."); + } + this.sampleAttributes.add(titleAttribute); + } + + public void addSampleAttribute(String title, SampleAttributeType sampleAttributeType, + boolean required, String linkedSampleTypeIdOrNull) { + SampleAttribute sampleAttribute = new SampleAttribute(title, sampleAttributeType, false, + required).withLinkedSampleTypeId(linkedSampleTypeIdOrNull); + sampleAttributes.add(sampleAttribute); + } + + public String getTitle() { + return title; + } + + public List getSample_attributes() { + return sampleAttributes; + } + } + + public static class SampleAttribute { + + private String title; + private String description; + private SampleAttributeType sampleAttributeType; + private boolean isTitle; + private boolean required; + private String linkedSampleTypeId; + + public SampleAttribute(String title, SampleAttributeType sampleAttributeType, boolean isTitle, + boolean required) { + this.title = title; + this.isTitle = isTitle; + this.required = required; + this.sampleAttributeType = sampleAttributeType; + } + + public SampleAttribute withDescription(String description) { + this.description = description; + return this; + } + + public SampleAttribute withLinkedSampleTypeId(String linkedSampleTypeId) { + this.linkedSampleTypeId = linkedSampleTypeId; + return this; + } + + public SampleAttributeType getSample_attribute_type() { + return sampleAttributeType; + } + + public String getLinked_sample_type_id() { + return linkedSampleTypeId; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public boolean getRequired() { + return required; + } + + public boolean getIs_title() { + return isTitle; + } + } + + public static class SampleAttributeType { + private String id; + private String title; + private String baseType; + + public SampleAttributeType(String id, String title, String baseType) { + this.id = id; + this.title = title; + this.baseType = baseType; + } + + public String getBase_type() { + return baseType; + } + + public String getId() { + return id; + } + + public String getTitle() { + return title; + } + } +} diff --git a/src/main/java/life/qbic/model/isa/ISAStudy.java b/src/main/java/life/qbic/model/isa/ISAStudy.java index 1fd19bf..7a30d5a 100644 --- a/src/main/java/life/qbic/model/isa/ISAStudy.java +++ b/src/main/java/life/qbic/model/isa/ISAStudy.java @@ -65,12 +65,6 @@ public Attributes getAttributes() { return attributes; } - public static void main(String[] args) throws JsonProcessingException { - ISAStudy study = new ISAStudy("title", "1"); - study.setCreatorIDs(Arrays.asList(3,2)); - System.err.println(study.toJson()); - } - private class Relationships { private String investigationId; diff --git a/src/main/java/life/qbic/model/isa/SeekStructure.java b/src/main/java/life/qbic/model/isa/SeekStructure.java index 323876c..9dd45b4 100644 --- a/src/main/java/life/qbic/model/isa/SeekStructure.java +++ b/src/main/java/life/qbic/model/isa/SeekStructure.java @@ -1,11 +1,8 @@ -package life.qbic.model; +package life.qbic.model.isa; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.util.List; import java.util.Map; -import life.qbic.model.isa.GenericSeekAsset; -import life.qbic.model.isa.ISAAssay; -import life.qbic.model.isa.ISASample; public class SeekStructure { From a87fed8f0931fb9003cc8b268d6b1a307c7d69c7 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 7 Oct 2024 16:43:13 +0200 Subject: [PATCH 4/4] small fixes and features --- .../io/commandline/AuthenticationOptions.java | 20 ++++- .../io/commandline/DownloadPetabCommand.java | 2 +- .../io/commandline/FindDatasetsCommand.java | 2 +- .../commandline/SampleHierarchyCommand.java | 2 +- .../commandline/SpaceStatisticsCommand.java | 2 +- .../TransferDataToSeekCommand.java | 6 +- .../TransferSampleTypesToSeekCommand.java | 6 +- .../io/commandline/UploadDatasetCommand.java | 2 +- .../commandline/UploadPetabResultCommand.java | 2 +- .../qbic/model/OpenbisSeekTranslator.java | 36 +++++++-- .../qbic/model/download/SEEKConnector.java | 73 +++++++++++++++---- 11 files changed, 116 insertions(+), 37 deletions(-) diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java index 1b9c5d0..eba85d1 100644 --- a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java +++ b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java @@ -8,6 +8,8 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; import java.util.StringJoiner; import java.util.TreeMap; import org.apache.logging.log4j.LogManager; @@ -29,13 +31,13 @@ public class AuthenticationOptions { @Option( names = {"-as", "-as_url"}, - description = "ApplicationServer URL", + description = "OpenBIS ApplicationServer URL", scope = CommandLine.ScopeType.INHERIT) private String as_url; @Option( names = {"-dss", "--dss_url"}, - description = "DatastoreServer URL", + description = "OpenBIS DatastoreServer URL", scope = CommandLine.ScopeType.INHERIT) private String dss_url; @@ -85,14 +87,14 @@ public String getSeekURL() { return seek_url; } - public String getDSS() { + public String getOpenbisDSS() { if(dss_url == null & configPath!=null && !configPath.isBlank()) { dss_url = ReadProperties.getProperties(configPath).get("dss"); } return dss_url; } - public String getAS() { + public String getOpenbisAS() { if(as_url == null & configPath!=null && !configPath.isBlank()) { as_url = ReadProperties.getProperties(configPath).get("as"); } @@ -107,6 +109,16 @@ public char[] getOpenbisPassword() { return openbisPasswordOptions.getPassword(); } + public String getOpenbisBaseURL() throws MalformedURLException { + URL asURL = new URL(as_url); + String res = asURL.getProtocol()+ "://" +asURL.getHost(); + if(asURL.getPort()!=-1) { + res+=":"+asURL.getPort(); + } + System.err.println(res); + return res; + } + /** * official picocli documentation example */ diff --git a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java index 46efefe..5be6b09 100644 --- a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java +++ b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java @@ -27,7 +27,7 @@ public class DownloadPetabCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index 2d340bd..9e384f9 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -39,7 +39,7 @@ public void run() { } else { System.out.println("Querying experiment in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.listDatasetsOfExperiment(spaces, experimentCode).stream() .sorted(Comparator.comparing( diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index 53b2bf3..2298693 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -43,7 +43,7 @@ public void run() { } else { summary.add("Querying samples in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); Map hierarchy = openbis.queryFullSampleHierarchy(spaces); diff --git a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java index ec2e2d0..6bed496 100644 --- a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java +++ b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java @@ -50,7 +50,7 @@ public void run() { } else { summary.add("Querying samples in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); if (spaces.isEmpty()) { diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index 72d8934..dbaeb67 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -76,7 +76,7 @@ public void run() { System.out.println("Connecting to openBIS..."); OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), - auth.getAS(), auth.getDSS()); + auth.getOpenbisAS(), auth.getOpenbisDSS()); openbis = new OpenbisConnector(authentication); @@ -106,8 +106,8 @@ public void run() { byte[] httpCredentials = Base64.encodeBase64( (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); try { - seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "seek_test", - "lisym default study"); + seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, auth.getOpenbisBaseURL(), + "seek_test", "lisym default study"); translator = seek.getTranslator(); } catch (URISyntaxException | IOException | InterruptedException e) { throw new RuntimeException(e); diff --git a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java index cdd6ee4..301f09a 100644 --- a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java @@ -28,7 +28,7 @@ public void run() { System.out.println("auth..."); OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), - auth.getAS(), auth.getDSS()); + auth.getOpenbisAS(), auth.getOpenbisDSS()); System.out.println("openbis..."); openbis = new OpenbisConnector(authentication); @@ -36,8 +36,8 @@ public void run() { byte[] httpCredentials = Base64.encodeBase64( (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); try { - seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "seek_test", - "lisym default study"); + seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, auth.getOpenbisBaseURL(), + "seek_test", "lisym default study"); translator = seek.getTranslator(); } catch (URISyntaxException | IOException | InterruptedException e) { throw new RuntimeException(e); diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index 9e2d3a9..a24a0c7 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -32,7 +32,7 @@ public class UploadDatasetCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); openbis = new OpenbisConnector(authentication); if(!pathValid(dataPath)) { diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 6ee72a9..8e36b1e 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -33,7 +33,7 @@ public class UploadPetabResultCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); openbis = new OpenbisConnector(authentication); if(!pathValid(dataPath)) { diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index 0504636..dfcc025 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -2,7 +2,6 @@ import static java.util.Map.entry; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.DataType; import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.PropertyAssignment; @@ -16,13 +15,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.ISAAssay; -import life.qbic.model.isa.ISADataFile; import life.qbic.model.isa.ISASample; import life.qbic.model.isa.ISASampleType; import life.qbic.model.isa.ISASampleType.SampleAttribute; @@ -34,12 +33,24 @@ public class OpenbisSeekTranslator { private final String DEFAULT_PROJECT_ID; private final String DEFAULT_STUDY_ID; private final String DEFAULT_TRANSFERRED_SAMPLE_TITLE = "openBIS Name"; + private final String openBISBaseURL; - public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { + public OpenbisSeekTranslator(String openBISBaseURL, String defaultProjectID, String defaultStudyID) { + this.openBISBaseURL = openBISBaseURL; this.DEFAULT_PROJECT_ID = defaultProjectID; this.DEFAULT_STUDY_ID = defaultStudyID; } + private String generateOpenBISLinkFromPermID(String entityType, String permID) { + StringBuilder builder = new StringBuilder(); + builder.append(openBISBaseURL); + builder.append("#entity="); + builder.append(entityType); + builder.append("&permId="); + builder.append(permID); + return builder.toString(); + } + Map experimentTypeToAssayClass = Map.ofEntries( entry("00_MOUSE_DATABASE", "EXP"), entry("00_PATIENT_DATABASE", "EXP"), @@ -63,8 +74,8 @@ public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { new SampleAttributeType("8", "String", "String")), entry(DataType.HYPERLINK, new SampleAttributeType("8", "String", "String")), entry(DataType.XML, new SampleAttributeType("7", "Text", "Text")), - entry(DataType.SAMPLE, //should be handled before mapping types - new SampleAttributeType("8", "String", "String")), + entry(DataType.SAMPLE, //we link the sample as URL to openBIS for now + new SampleAttributeType("5", "Web link", "String")), entry(DataType.DATE, new SampleAttributeType("2", "Date time", "Date")) ); @@ -150,12 +161,23 @@ public SeekStructure translate(OpenbisExperimentWithDescendants experiment, //try to put all attributes into sample properties, as they should be a 1:1 mapping Map typeCodesToNames = new HashMap<>(); + Set propertiesLinkingSamples = new HashSet<>(); for (PropertyAssignment a : sampleType.getPropertyAssignments()) { - typeCodesToNames.put(a.getPropertyType().getCode(), a.getPropertyType().getLabel()); + String code = a.getPropertyType().getCode(); + String label = a.getPropertyType().getLabel(); + DataType type = a.getPropertyType().getDataType(); + typeCodesToNames.put(code, label); + if(type.equals(DataType.SAMPLE)) { + propertiesLinkingSamples.add(code); + } } Map attributes = new HashMap<>(); for(String code : sample.getProperties().keySet()) { - attributes.put(typeCodesToNames.get(code), sample.getProperties().get(code)); + String value = sample.getProperty(code); + if(propertiesLinkingSamples.contains(code)) { + value = generateOpenBISLinkFromPermID("SAMPLE", value); + } + attributes.put(typeCodesToNames.get(code), value); } attributes.put(DEFAULT_TRANSFERRED_SAMPLE_TITLE, sample.getIdentifier().getIdentifier()); diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java index 580bf0f..16fb0ec 100644 --- a/src/main/java/life/qbic/model/download/SEEKConnector.java +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -30,7 +30,6 @@ import life.qbic.model.isa.ISASample; import life.qbic.model.isa.ISASampleType; import life.qbic.model.isa.ISAStudy; -import org.apache.commons.codec.binary.Base64; import org.apache.http.client.utils.URIBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -41,13 +40,56 @@ public class SEEKConnector { private String apiURL; private byte[] credentials; private OpenbisSeekTranslator translator; + private final String DEFAULT_PROJECT_ID; - public SEEKConnector(String apiURL, byte[] httpCredentials, String defaultProjectTitle, - String defaultStudyTitle) throws URISyntaxException, IOException, InterruptedException { + public SEEKConnector(String apiURL, byte[] httpCredentials, String openBISBaseURL, + String defaultProjectTitle, String defaultStudyTitle) throws URISyntaxException, IOException, + InterruptedException { this.apiURL = apiURL; this.credentials = httpCredentials; - translator = new OpenbisSeekTranslator("1", //searchNodeWithTitle("projects", defaultProjectTitle), - "1");//searchNodeWithTitle("studies", defaultStudyTitle)); + Optional projectID = getProjectWithTitle(defaultProjectTitle); + if(projectID.isEmpty()) { + throw new RuntimeException("Failed to find project with title: " + defaultProjectTitle+". " + + "Please provide an existing default project."); + } + DEFAULT_PROJECT_ID = projectID.get(); + + translator = new OpenbisSeekTranslator(openBISBaseURL, DEFAULT_PROJECT_ID, + searchNodeWithTitle("studies", defaultStudyTitle)); + } + + /** + * Lists projects and returns the optional identifier of the one matching the provided ID. + * Necessary because project search does not seem to work. + * @param projectTitle the title to search for + * @return + */ + private Optional getProjectWithTitle(String projectTitle) + throws IOException, InterruptedException, URISyntaxException { + String endpoint = apiURL+"/projects/"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode hits = rootNode.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + String id = hit.get("id").asText(); + String title = hit.get("attributes").get("title").asText(); + if(title.equals(projectTitle)) { + return Optional.of(id); + } + } + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + return Optional.empty(); } public String addAssay(ISAAssay assay) @@ -231,13 +273,16 @@ public String uploadStreamContent(String blobEndpoint, HttpRequest request = HttpRequest.newBuilder() .uri(new URI(blobEndpoint)) .headers("Content-Type", "application/octet-stream") - .headers("Accept", "application/octet-stream") + .headers("Accept", "*/*") .headers("Authorization", "Basic " + new String(credentials)) .PUT(BodyPublishers.ofInputStream(streamSupplier)).build(); HttpResponse response = HttpClient.newBuilder().build() .send(request, BodyHandlers.ofString()); + System.err.println("response was: "+response); + System.err.println("response body: "+response.body()); + if(response.statusCode()!=200) { System.err.println(response.body()); throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); @@ -276,9 +321,9 @@ public Optional createAssetForFile(DataSetFile file, String asset if(!file.getPath().isBlank() && !file.isDirectory()) { File f = new File(file.getPath()); String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode+": "+f.getName(); - GenericSeekAsset isaFile = new GenericSeekAsset("data_files", assetName, file.getPath(), - Arrays.asList("1"));//TODO + String assetName = datasetCode+": "+f.getName();//TODO what do we want to call the asset? + GenericSeekAsset isaFile = new GenericSeekAsset(assetType, assetName, file.getPath(), + Arrays.asList(DEFAULT_PROJECT_ID)); isaFile.withAssays(assays); String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); String annotation = translator.dataFormatAnnotationForExtension(fileExtension); @@ -298,9 +343,9 @@ public List createAssets(List filesInDataset, if(!file.getPath().isBlank() && !file.isDirectory()) { File f = new File(file.getPath()); String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode+": "+f.getName();//TODO? + String assetName = datasetCode+": "+f.getName();//TODO what do we want to call the asset? GenericSeekAsset isaFile = new GenericSeekAsset("data_files", assetName, file.getPath(), - Arrays.asList("1")); + Arrays.asList(DEFAULT_PROJECT_ID)); isaFile.withAssays(assays); String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); String annotation = translator.dataFormatAnnotationForExtension(fileExtension); @@ -384,7 +429,7 @@ private Map parseSampleTypesJSON(String json) throws JsonProcess private String searchNodeWithTitle(String nodeType, String title) throws URISyntaxException, IOException, InterruptedException { - String endpoint = apiURL+"/search/"; + String endpoint = apiURL+"/search"; URIBuilder builder = new URIBuilder(endpoint); builder.setParameter("q", title).setParameter("search_type", nodeType); @@ -402,8 +447,7 @@ private String searchNodeWithTitle(String nodeType, String title) JsonNode hits = rootNode.path("data"); for (Iterator it = hits.elements(); it.hasNext(); ) { JsonNode hit = it.next(); - System.err.println(hit.asText()); - if(hit.get("title").asText().equals(title)) { + if(hit.get("attributes").get("title").asText().equals(title)) { return hit.get("id").asText(); } } @@ -458,6 +502,7 @@ public AssayWithQueuedAssets createNode(SeekStructure nodeWithChildren, boolean throws URISyntaxException, IOException, InterruptedException { String assayID = addAssay(nodeWithChildren.getAssay()); for(ISASample sample : nodeWithChildren.getSamples()) { + sample.setAssayIDs(Arrays.asList(assayID)); createSample(sample); }