Merge pull request #6 from qbicsoftware/feature/petab-basic-features

Petab download and upload

Author: wow-such-code

pom.xml

@@ -94,6 +94,11 @@
       <artifactId>picocli</artifactId>
       <version>4.6.2</version>
     </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-yaml</artifactId>
+      <version>2.17.2</version>
+    </dependency>
     <dependency>
       <groupId>jline</groupId>
       <artifactId>jline</artifactId>

src/main/java/life/qbic/io/PetabParser.java

@@ -0,0 +1,97 @@
+package life.qbic.io;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import life.qbic.model.petab.PetabMetadata;
+
+public class PetabParser {
+
+  private final String META_INFO_YAML = "metaInformation.yaml";
+
+  public PetabMetadata parse(String dataPath) {
+
+    File directory = new File(dataPath);
+    List<String> sourcePetabReferences = new ArrayList<>();
+
+    File yaml = findYaml(directory);
+    if (yaml != null) {
+      BufferedReader reader = null;
+      try {
+        reader = new BufferedReader(new FileReader(yaml));
+        boolean inIDBlock = false;
+        while (true) {
+          String line = reader.readLine();
+          if (line == null) {
+            break;
+          }
+          // the id block ends, when a new key with colon is found
+          if (inIDBlock && line.contains(":")) {
+            inIDBlock = false;
+          }
+          // if we are in the id block, we collect one dataset code per line
+          if (inIDBlock) {
+            parseDatasetCode(line).ifPresent(sourcePetabReferences::add);
+          }
+          if (line.contains("openBISSourceIds:")) {
+            inIDBlock = true;
+          }
+        }
+        reader.close();
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+    return new PetabMetadata(sourcePetabReferences);
+  }
+
+  private Optional<String> parseDatasetCode(String line) {
+    // expected input: "    - 20240702093837370-684137"
+    String[] tokens = line.split("-");
+    if(tokens.length == 3) {
+      return Optional.of(tokens[1].strip()+"-"+tokens[2].strip());
+    } else {
+      System.out.println("Could not extract dataset code from the following line:");
+      System.out.println(line);
+    }
+    return Optional.empty();
+  }
+
+  public void addDatasetId(String outputPath, String datasetCode) throws IOException {
+
+    Path path = Paths.get(Objects.requireNonNull(findYaml(new File(outputPath))).getPath());
+    Charset charset = StandardCharsets.UTF_8;
+
+    final String keyWord = "openBISId";
+
+    String idInLine = keyWord+":(.*)?(\\r\\n|[\\r\\n])";
+
+    String content = Files.readString(path, charset);
+    content = content.replaceAll(idInLine, keyWord+": "+datasetCode+"\n");
+    Files.write(path, content.getBytes(charset));
+  }
+
+  private File findYaml(File directory) {
+    for (File file : Objects.requireNonNull(directory.listFiles())) {
+      if (file.isFile() && file.getName().equalsIgnoreCase(META_INFO_YAML)) {
+        return file;
+      }
+      if (file.isDirectory()) {
+        return findYaml(file);
+      }
+    }
+    System.out.println(META_INFO_YAML + " not found");
+    return null;
+  }
+
+}

src/main/java/life/qbic/io/commandline/CommandLineOptions.java

@@ -8,8 +8,8 @@
 
 // main command with format specifiers for the usage help message
 @Command(name = "openbis-scripts",
-        subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class,
-            UploadDatasetCommand.class, SpaceStatisticsCommand.class },
+        subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, DownloadPetabCommand.class,
+            UploadPetabResultCommand.class, UploadDatasetCommand.class, SpaceStatisticsCommand.class },
         description = "A client software for querying openBIS.",
         mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class)
 public class CommandLineOptions {

src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java

@@ -0,0 +1,60 @@
+package life.qbic.io.commandline;
+
+import ch.ethz.sis.openbis.generic.OpenBIS;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import life.qbic.App;
+import life.qbic.io.PetabParser;
+import life.qbic.model.DatasetWithProperties;
+import life.qbic.model.download.OpenbisConnector;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Mixin;
+import picocli.CommandLine.Parameters;
+
+@Command(name = "download-petab",
+    description = "Downloads PEtab dataset and stores some additional information from openbis in the petab.yaml")
+public class DownloadPetabCommand implements Runnable {
+
+  @Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset to download. Can be found via list-data.")
+  private String datasetCode;
+  @Parameters(arity = "1", paramLabel = "download path", description = "The local path where to store the downloaded data")
+  private String outputPath;
+  @Mixin
+  AuthenticationOptions auth = new AuthenticationOptions();
+
+    @Override
+    public void run() {
+      OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS());
+      OpenbisConnector openbis = new OpenbisConnector(authentication);
+
+      List<DataSet> datasets = openbis.findDataSets(Collections.singletonList(datasetCode));
+
+      if(datasets.isEmpty()) {
+        System.out.println(datasetCode+" not found");
+        return;
+      }
+      DatasetWithProperties result = new DatasetWithProperties(datasets.get(0));
+      Optional<String> patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID",
+          result.getExperiment().getIdentifier());
+      patientID.ifPresent(s -> result.addProperty("patientID", s));
+
+      System.out.println("Found dataset, downloading.");
+      System.out.println();
+
+      openbis.downloadDataset(outputPath, datasetCode);
+
+      PetabParser parser = new PetabParser();
+      try {
+        System.out.println("Adding dataset identifier to metaInformation.yaml.");
+        parser.addDatasetId(outputPath, datasetCode);
+      } catch (IOException e) {
+        System.out.println("Could not add dataset identifier.");
+        throw new RuntimeException(e);
+      }
+      System.out.println("Done");
+    }
+
+}

src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java

@@ -70,9 +70,4 @@ private boolean pathValid(String dataPath) {
       return new File(dataPath).exists();
   }
 
-  private String getTimeStamp() {
-      final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss";
-      DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT);
-      return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter);
-    }
 }

src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java

@@ -0,0 +1,79 @@
+package life.qbic.io.commandline;
+
+import ch.ethz.sis.openbis.generic.OpenBIS;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId;
+import java.io.File;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import life.qbic.App;
+import life.qbic.io.PetabParser;
+import life.qbic.model.download.OpenbisConnector;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Mixin;
+import picocli.CommandLine.Parameters;
+
+@Command(name = "upload-petab",
+    description = "uploads a PETab folder and attaches it to a provided experiment and any datasets referenced in the PETab metadata (e.g. for PETab results).")
+public class UploadPetabResultCommand implements Runnable {
+
+  @Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder to upload")
+  private String dataPath;
+  @Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the experiment the data should be attached to. "
+      + "The identifier must be of the format: /space/project/experiment")
+  private String experimentID;
+  //@Option(arity = "1..*", paramLabel = "<parent_datasets>", description = "Optional list of dataset codes to act"
+  //    + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"})
+  private List<String> parents = new ArrayList<>();
+  @Mixin
+  AuthenticationOptions auth = new AuthenticationOptions();
+
+  private OpenbisConnector openbis;
+  private PetabParser petabParser = new PetabParser();
+
+    @Override
+    public void run() {
+      OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS());
+      openbis = new OpenbisConnector(authentication);
+
+      if(!pathValid(dataPath)) {
+        System.out.printf("Path %s could not be found%n", dataPath);
+        return;
+      }
+      if(!experimentExists(experimentID)) {
+        System.out.printf("Experiment %s could not be found%n", experimentID);
+        return;
+      }
+      System.out.println("Looking for reference datasets in metaInformation.yaml...");
+      parents = petabParser.parse(dataPath).getSourcePetabReferences();
+      if(parents.isEmpty()) {
+        System.out.println("No reference datasets found in openBISSourceIds property. Assuming"
+            + "this is a new dataset.");
+      } else {
+        System.out.println("Found reference ids: " + String.join(", ", parents));
+        if (!datasetsExist(parents)) {
+          System.out.printf("One or more datasets %s could not be found%n", parents);
+          return;
+        } else {
+          System.out.println("Referenced datasets found");
+        }
+      }
+      System.out.println("Uploading dataset...");
+      //TODO copy and remove source references here
+      DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents);
+      System.out.printf("Dataset %s was successfully created%n", result.getPermId());
+    }
+
+  private boolean datasetsExist(List<String> datasetCodes) {
+      return openbis.findDataSets(datasetCodes).size() == datasetCodes.size();
+  }
+
+  private boolean experimentExists(String experimentID) {
+      return openbis.experimentExists(experimentID);
+  }
+
+  private boolean pathValid(String dataPath) {
+      return new File(dataPath).exists();
+  }
+
+}

src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java

@@ -0,0 +1,16 @@
+package life.qbic.model.petab;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+
+public class Arguments {
+  @JsonProperty
+  List<String> housekeeperObservableIds;
+
+  @Override
+  public String toString() {
+    return "Arguments{" +
+        "housekeeperObservableIds=" + housekeeperObservableIds +
+        '}';
+  }
+}

src/main/java/life/qbic/model/petab/Arguments.java

@@ -0,0 +1,23 @@
+package life.qbic.model.petab;
+
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class CellCountInfo {
+  @JsonProperty
+  double seeded;
+  @JsonProperty
+  String ncellsCount;
+  @JsonProperty
+  String unit;
+
+  @Override
+  public String toString() {
+    return "CellCountInfo{" +
+        "seeded=" + seeded +
+        ", ncellsCount='" + ncellsCount + '\'' +
+        ", unit='" + unit + '\'' +
+        '}';
+  }
+}
+