Merge pull request #13 from qbicsoftware/feature/sample-and-dataset-t…

…ransfer

changes to seek transfer and more comments

src/main/java/life/qbic/io/PetabParser.java

@@ -18,7 +18,7 @@
 
 public class PetabParser {
 
-  private final String META_INFO_YAML = "metaInformation.yaml";
+  private final String META_INFO_YAML_NAME = "metaInformation";
 
   public PetabMetadata parse(String dataPath) {
 
@@ -84,14 +84,15 @@ public void addDatasetId(String outputPath, String datasetCode) throws IOExcepti
 
   private File findYaml(File directory) {
     for (File file : Objects.requireNonNull(directory.listFiles())) {
-      if (file.isFile() && file.getName().equalsIgnoreCase(META_INFO_YAML)) {
+      String fileName = file.getName();
+      if (file.isFile() && fileName.contains(META_INFO_YAML_NAME) && fileName.endsWith(".yaml")) {
         return file;
       }
       if (file.isDirectory()) {
         return findYaml(file);
       }
     }
-    System.out.println(META_INFO_YAML + " not found");
+    System.out.println(META_INFO_YAML_NAME + " yaml not found.");
     return null;
   }
 

src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java

@@ -5,7 +5,6 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
-import java.util.Optional;
 import java.util.Set;
 import life.qbic.App;
 import life.qbic.io.PetabParser;
@@ -15,13 +14,25 @@
 import picocli.CommandLine.Mixin;
 import picocli.CommandLine.Parameters;
 
+/**
+ * The Download PEtab command can be used to download a PEtab Dataset from openBIS and store some
+ * additional information from openBIS in the metaInformation.yaml file (or a respective yaml file
+ * containing 'metaInformation' in its name).
+ * The Dataset to download is specified by providing the openBIS dataset identifier (code) and the
+ * PEtab is downloaded to the download path parameter provided.
+ * By design, the Dataset Identifier is added to the downloaded metaInformation.yaml as 'openBISId'
+ * in order to keep track of the source of this PEtab.
+ */
 @Command(name = "download-petab",
-    description = "Downloads PEtab dataset and stores some additional information from openbis in the petab.yaml")
+    description = "Downloads PEtab dataset and stores some additional information from openbis in "
+        + "the petab.yaml")
 public class DownloadPetabCommand implements Runnable {
 
-  @Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset to download. Can be found via list-data.")
+  @Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset to "
+      + "download. Can be found via list-data.")
   private String datasetCode;
-  @Parameters(arity = "1", paramLabel = "download path", description = "The local path where to store the downloaded data")
+  @Parameters(arity = "1", paramLabel = "download path", description = "The local path where to "
+      + "store the downloaded data")
   private String outputPath;
   @Mixin
   OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions();

src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java

@@ -19,16 +19,22 @@
 import picocli.CommandLine.Option;
 import picocli.CommandLine.Parameters;
 
+/**
+ * List Data
+ * The list-data command can be used to list Datasets in openBIS and some of their metadata based
+ * on the experiment or sample they are attached to. Experiments or samples are specified by their
+ * openBIS code or identifier.
+ * The optional 'space' parameter can be used to only show datasets found in the provided space.
+ */
 @Command(name = "list-data",
     description = "lists datasets and their details for a given experiment code")
 public class FindDatasetsCommand implements Runnable {
 
-  @Parameters(arity = "1", paramLabel = "openBIS obejct", description =
-      "The code of the experiment "
-          + "or sample data is attached to.")
+  @Parameters(arity = "1", paramLabel = "openBIS object", description =
+      "The code of the experiment or sample data is attached to.")
   private String objectCode;
   @Option(arity = "1", paramLabel = "<space>", description = "Optional openBIS spaces to filter "
-      + "found datasets by", names = {"-s", "--space"})
+      + "found datasets by.", names = {"-s", "--space"})
   private String space;
   @Mixin
   OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions();

src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java

@@ -22,13 +22,22 @@
 import picocli.CommandLine.Mixin;
 import picocli.CommandLine.Option;
 
+/**
+ * The Sample Types command queries all sample types and prints which types are connected and how
+ * often (via samples existing in the queried openBIS instance), creating a sample type hierarchy.
+ * The --space command can be used to only show the sample-types used in a specific openBIS space.
+ * An output file for the resulting hierarchy can be specified using the --out command.
+ */
 @Command(name = "sample-types",
-    description = "lists sample types with children sample types and how often they are found in the openbis instance")
+    description = "lists sample types with children sample types and how often they are found in "
+        + "the openbis instance")
 public class SampleHierarchyCommand implements Runnable {
 
-  @Option(arity = "1", paramLabel = "<space>", description = "optional openBIS spaces to filter samples", names = {"-s", "--space"})
+  @Option(arity = "1", paramLabel = "<space>", description = "optional openBIS space to filter "
+      + "samples", names = {"-s", "--space"})
   private String space;
-  @Option(arity = "1", paramLabel = "<output file path>", description = "optional output path", names = {"-o", "--out"})
+  @Option(arity = "1", paramLabel = "<output file path>", description = "optional output path",
+      names = {"-o", "--out"})
   private String outpath;
   @Mixin
   OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions();
@@ -44,8 +53,8 @@ public void run() {
         } else {
           summary.add("Querying samples in all available spaces...");
         }
-        OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(),
-            auth.getOpenbisAS());
+        OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(),
+            auth.getOpenbisUser(), auth.getOpenbisAS());
         OpenbisConnector openbis = new OpenbisConnector(authentication);
         Map<SampleTypeConnection, Integer> hierarchy = openbis.queryFullSampleHierarchy(spaces);
 
@@ -57,7 +66,7 @@ public void run() {
         System.out.println(s);
       }
       Path outputPath = Paths.get(Configuration.LOG_PATH.toString(),
-          "sample_model_summary"+getTimeStamp()+".txt");
+          "sample_model_summary" + getTimeStamp() + ".txt");
       if(outpath!=null) {
         outputPath = Paths.get(outpath);
       }
@@ -72,6 +81,6 @@ public void run() {
     private String getTimeStamp() {
       final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss";
       DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT);
-      return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter).toString();
+      return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter);
     }
 }

src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java

@@ -24,16 +24,27 @@
 import picocli.CommandLine.Mixin;
 import picocli.CommandLine.Option;
 
+/**
+ * The Statistics command can be used to list the number of collections, sample objects and attached
+ * datasets by type for one or all spaces accessible by the user.
+ * The --space command can be used to only show the objects in a specific openBIS space.
+ * An output file for the resulting list can be specified using the --out command.
+ * By default, openBIS settings objects and material spaces are ignored. This can be overwritten
+ * using --show-settings.
+ */
 @Command(name = "statistics",
     description = "lists the number of collections, sample objects and attached datasets (by type)"
         + "for one or all spaces accessible by the user")
 public class SpaceStatisticsCommand implements Runnable {
 
-  @Option(arity = "1", paramLabel = "<space>", description = "optional openBIS spaces to filter samples", names = {"-s", "--space"})
+  @Option(arity = "1", paramLabel = "<space>", description = "optional openBIS space to filter "
+      + "samples", names = {"-s", "--space"})
   private String space;
-  @Option(arity = "1", paramLabel = "<output file path>", description = "optional output path", names = {"-o", "--out"})
+  @Option(arity = "1", paramLabel = "<output file path>", description = "optional output path",
+      names = {"-o", "--out"})
   private String outpath;
-  @Option(arity = "0", description = "shows results for openBIS settings and material spaces. Ignored if a specific space is selected.",
+  @Option(arity = "0", description = "shows results for openBIS settings and material spaces. "
+      + "Ignored if a specific space is selected.",
       names = {"--show-settings"})
   private boolean allSpaces;
   @Mixin

src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java

@@ -3,7 +3,6 @@
 import ch.ethz.sis.openbis.generic.OpenBIS;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
-import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile;
 import java.io.File;
 import java.io.IOException;
 import java.net.URISyntaxException;
@@ -19,18 +18,14 @@
 import java.util.stream.Stream;
 import javax.xml.parsers.ParserConfigurationException;
 import life.qbic.App;
-import life.qbic.model.DatasetWithProperties;
 import life.qbic.model.OpenbisExperimentWithDescendants;
-import life.qbic.model.OpenbisSampleWithDatasets;
 import life.qbic.model.OpenbisSeekTranslator;
 import life.qbic.model.download.SEEKConnector.SeekStructurePostRegistrationInformation;
-import life.qbic.model.isa.GenericSeekAsset;
 import life.qbic.model.isa.SeekStructure;
 import life.qbic.model.download.OpenbisConnector;
 import life.qbic.model.download.SEEKConnector;
 import life.qbic.model.download.SEEKConnector.AssetToUpload;
 import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang3.tuple.Pair;
 import org.xml.sax.SAXException;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Mixin;
@@ -152,13 +147,18 @@ public void run() {
       throw new RuntimeException(e);
     }
     SeekStructurePostRegistrationInformation postRegInfo;
+    OpenbisExperimentWithDescendants structure;
     try {
+      System.out.println("Collecting information from openBIS...");
       if (isExperiment) {
-        postRegInfo = handleExperimentTransfer();
+        structure = openbis.getExperimentWithDescendants(objectID);
+        postRegInfo = handleExperimentTransfer(structure);
       } else if (isSample) {
-        postRegInfo = handleSampleTransfer();
+        structure = openbis.getExperimentAndDataFromSample(objectID);
+        postRegInfo = handleExperimentTransfer(structure);
       } else {
-        postRegInfo = handleDatasetTransfer();
+        structure = openbis.getExperimentStructureFromDataset(objectID);
+        postRegInfo = handleExperimentTransfer(structure);
       }
     } catch (URISyntaxException | IOException | InterruptedException e) {
       throw new RuntimeException(e);
@@ -170,10 +170,9 @@ public void run() {
     System.out.println("Done");
   }
 
-  private SeekStructurePostRegistrationInformation handleExperimentTransfer()
+  private SeekStructurePostRegistrationInformation handleExperimentTransfer(
+      OpenbisExperimentWithDescendants experiment)
       throws URISyntaxException, IOException, InterruptedException {
-    System.out.println("Collecting information from openBIS...");
-    OpenbisExperimentWithDescendants experiment = openbis.getExperimentWithDescendants(objectID);
     Set<String> blacklist = parseBlackList(blacklistFile);
     System.out.println("Translating openBIS property codes to SEEK names...");
     Map<String, String> sampleTypesToIds = seek.getSampleTypeNamesToIDs();
@@ -196,10 +195,10 @@ private SeekStructurePostRegistrationInformation handleExperimentTransfer()
     return createNewAssayStructure(nodeWithChildren);
   }
 
+  /*
   private SeekStructurePostRegistrationInformation handleSampleTransfer()
       throws URISyntaxException, IOException, InterruptedException {
     System.out.println("Collecting information from openBIS...");
-    OpenbisSampleWithDatasets sampleWithDatasets = openbis.getSampleWithDatasets(objectID);
     Set<String> blacklist = parseBlackList(blacklistFile);
     System.out.println("Translating openBIS property codes to SEEK names...");
     Map<String, String> sampleTypesToIds = seek.getSampleTypeNamesToIDs();
@@ -235,6 +234,7 @@ private SeekStructurePostRegistrationInformation handleDatasetTransfer()
     return createNewAssetsForDataset(nodeWithChildren.getISAFileToDatasetFiles());
   }
 
+
   private SeekStructurePostRegistrationInformation updateSampleStructure(
       SeekStructure nodeWithChildren, String sampleID)
       throws URISyntaxException, IOException, InterruptedException {
@@ -277,6 +277,8 @@ private SeekStructurePostRegistrationInformation createNewAssetsForDataset(
     return postRegistrationInformation;
   }
 
+   */
+
   private Set<String> parseBlackList(String blacklistFile) {
     if(blacklistFile == null) {
       return new HashSet<>();

src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java

@@ -13,6 +13,17 @@
 import picocli.CommandLine.Option;
 import picocli.CommandLine.Parameters;
 
+/**
+ * The Upload Dataset command can be used to upload a Dataset to openBIS and connect it to existing
+ * datasets.
+ * To upload a dataset, the path to the file or folder and the object ID to which it should
+ * be attached need to be provided. Objects can be experiments or samples.
+ * Parent datasets can be specified using the --parents command.
+ * If the specified object ID or any of the specified parent datasets cannot be found, the script
+ * will stop and return an error message.
+ * The dataset type of the new dataset in openBIS can be specified using the --type option,
+ * otherwise the type "UNKNOWN" will be used.
+ */
 @Command(name = "upload-data",
     description = "uploads a dataset and attaches it to an experiment and (optionally) other datasets")
 public class UploadDatasetCommand implements Runnable {

src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java

@@ -4,7 +4,6 @@
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId;
 import java.io.File;
 import java.nio.file.Path;
-import java.util.ArrayList;
 import java.util.List;
 import life.qbic.App;
 import life.qbic.io.PetabParser;
@@ -14,12 +13,29 @@
 import picocli.CommandLine.Option;
 import picocli.CommandLine.Parameters;
 
+/**
+ * The Upload PEtab command can be used to upload a PEtab Dataset to openBIS and connect it to its
+ * source files if these are stored in the same openBIS instance and referenced in the PEtabs meta-
+ * data.
+ * To upload a PEtab dataset, the path to the PEtab folder and the experiment ID to which it should
+ * be attached need to be provided.
+ * The dataset type of the new dataset in openBIS can be specified using the --type option,
+ * otherwise the type "UNKNOWN" will be used.
+ * The script will search the metaInformation.yaml for the entry "openBISSourceIds:" and attach the
+ * new dataset to all the datasets with ids in the following blocks found in this instance of
+ * openBIS:
+ *   openBISSourceIds:
+ *     - 20210702093837370-184137
+ *     - 20220702100912333-189138
+ * If one or more dataset identifiers are not found, the script will stop without uploading the data
+ * and inform the user.
+ */
 @Command(name = "upload-petab",
     description = "uploads a PETab folder and attaches it to a provided experiment and any datasets "
         + "referenced in the PETab metadata (e.g. for PETab results).")
 public class UploadPetabResultCommand implements Runnable {
 
-  @Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder "
+  @Parameters(arity = "1", paramLabel = "PEtab folder", description = "The path to the PEtab folder "
       + "to upload")
   private String dataPath;
   @Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the "
@@ -33,7 +49,7 @@ public class UploadPetabResultCommand implements Runnable {
   OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions();
 
   private OpenbisConnector openbis;
-  private PetabParser petabParser = new PetabParser();
+  private final PetabParser petabParser = new PetabParser();
 
     @Override
     public void run() {

src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java

@@ -1,6 +1,5 @@
 package life.qbic.model;
 
-import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
 import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile;

src/main/java/life/qbic/model/OpenbisSeekTranslator.java

@@ -218,6 +218,7 @@ public SeekStructure translate(OpenbisExperimentWithDescendants experiment,
     return result;
   }
 
+  /*
   public SeekStructure translate(OpenbisSampleWithDatasets sampleWithDatasets,
       Map<String, String> sampleTypesToIds, Set<String> blacklist, boolean transferData) {
     Sample sample = sampleWithDatasets.getSample();
@@ -281,6 +282,8 @@ public SeekStructure translate(Pair<DatasetWithProperties, List<DataSetFile>> da
     return new SeekStructure(assetToDatasetFiles);
   }
 
+   */
+
   private String getDatasetTypeOfFile(DataSetFile file, List<DatasetWithProperties> dataSets) {
     String permId = file.getDataSetPermId().getPermId();
     for(DatasetWithProperties dataset : dataSets) {