diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index 3acf00a..f2d1538 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -8,7 +8,8 @@ // main command with format specifiers for the usage help message @Command(name = "openbis-scripts", - subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, UploadDatasetCommand.class }, + subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, + UploadDatasetCommand.class, SpaceStatisticsCommand.class }, description = "A client software for querying openBIS.", mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) public class CommandLineOptions { diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index 82c2a18..2846fe7 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -4,15 +4,15 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; import ch.ethz.sis.openbis.generic.asapi.v3.dto.person.Person; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import life.qbic.App; +import life.qbic.model.DatasetWithProperties; import life.qbic.model.download.OpenbisConnector; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; @@ -45,12 +45,28 @@ public void run() { .sorted(Comparator.comparing( (DataSet d) -> d.getExperiment().getProject().getSpace().getCode())).collect( Collectors.toList()); + Map properties = new HashMap<>(); + if (!datasets.isEmpty()) { + Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", + datasets.get(0).getExperiment().getIdentifier()); + patientID.ifPresent(s -> properties.put("Patient ID", s)); + } + List datasetWithProperties = datasets.stream().map(dataSet -> { + DatasetWithProperties ds = new DatasetWithProperties(dataSet); + for (String key : properties.keySet()) { + ds.addProperty(key, properties.get(key)); + } + return ds; + }).collect(Collectors.toList()); int datasetIndex = 0; System.out.println(); System.out.printf("Found %s datasets for experiment %s:%n", datasets.size(), experimentCode); - for (DataSet dataSet : datasets) { + for (DatasetWithProperties dataSet : datasetWithProperties) { datasetIndex++; System.out.println("["+datasetIndex+"]"); + for(String key : dataSet.getProperties().keySet()) { + System.out.println(key+ ": "+properties.get(key)); + } System.out.printf("ID: %s (%s)%n", dataSet.getCode(), dataSet.getExperiment().getIdentifier()); System.out.println("Type: "+dataSet.getType().getCode()); Person person = dataSet.getRegistrator(); @@ -62,9 +78,4 @@ public void run() { } } - private String getTimeStamp() { - final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; - DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); - return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter); - } } diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index e9668d6..47353b6 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -16,7 +16,7 @@ import life.qbic.model.Configuration; import life.qbic.model.SampleTypeConnection; import life.qbic.model.download.FileSystemWriter; -import life.qbic.model.download.ModelReporter; +import life.qbic.model.download.SummaryWriter; import life.qbic.model.download.OpenbisConnector; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; @@ -38,10 +38,10 @@ public void run() { List summary = new ArrayList<>(); List spaces = new ArrayList<>(); if(space!=null) { - summary.add("Querying samples in space: "+space+"...\n"); + summary.add("Querying samples in space: "+space+"..."); spaces.add(space); } else { - summary.add("Querying samples in all available spaces...\n"); + summary.add("Querying samples in all available spaces..."); } OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); @@ -55,13 +55,13 @@ public void run() { System.out.println(s); } Path outputPath = Paths.get(Configuration.LOG_PATH.toString(), - "summary_model_"+getTimeStamp()+".txt"); + "sample_model_summary"+getTimeStamp()+".txt"); if(outpath!=null) { outputPath = Paths.get(outpath); } - ModelReporter modelReporter = new FileSystemWriter(outputPath); + SummaryWriter summaryWriter = new FileSystemWriter(outputPath); try { - modelReporter.reportSummary(summary); + summaryWriter.reportSummary(summary); } catch (IOException e) { throw new RuntimeException("Could not write summary file."); } diff --git a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java new file mode 100644 index 0000000..bce2144 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java @@ -0,0 +1,135 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import life.qbic.App; +import life.qbic.model.Configuration; +import life.qbic.model.download.FileSystemWriter; +import life.qbic.model.download.SummaryWriter; +import life.qbic.model.download.OpenbisConnector; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; + +@Command(name = "statistics", + description = "lists the number of collections, sample objects and attached datasets (by type)" + + "for one or all spaces accessible by the user") +public class SpaceStatisticsCommand implements Runnable { + + @Option(arity = "1", paramLabel = "", description = "optional openBIS spaces to filter samples", names = {"-s", "--space"}) + private String space; + @Option(arity = "1", paramLabel = "", description = "optional output path", names = {"-o", "--out"}) + private String outpath; + @Option(arity = "0", description = "shows results for openBIS settings and material spaces. Ignored if a specific space is selected.", + names = {"--show-settings"}) + private boolean allSpaces; + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + + @Override + public void run() { + List summary = new ArrayList<>(); + List blackList = new ArrayList<>(Arrays.asList("ELN_SETTINGS", "MATERIAL.GLOBAL")); + List spaces = new ArrayList<>(); + if (space != null) { + summary.add("Querying samples in space: " + space); + spaces.add(space); + } else { + summary.add("Querying samples in all available spaces..."); + } + OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); + OpenbisConnector openbis = new OpenbisConnector(authentication); + + if (spaces.isEmpty()) { + spaces = openbis.getSpaces(); + if(!allSpaces) { + spaces.removeAll(blackList); + } + } + + Map>> experiments = openbis.getExperimentsByTypeAndSpace(spaces); + Map>> samples = openbis.getSamplesByTypeAndSpace(spaces); + Map>> datasets = openbis.getDatasetsByTypeAndSpace(spaces); + + for(String space : spaces) { + summary.add("-----"); + summary.add("Summary for "+space); + summary.add("-----"); + int numExps = 0; + if (experiments.containsKey(space)) { + numExps = experiments.get(space).values().stream().mapToInt(List::size).sum(); + } + summary.add("Experiments ("+numExps+"):"); + summary.add(""); + if(!experiments.isEmpty()) { + Map> exps = experiments.get(space); + for (String type : exps.keySet()) { + summary.add(type + ": " + exps.get(type).size()); + } + } + summary.add(""); + int numSamples = 0; + if (samples.containsKey(space)) { + numSamples = samples.get(space).values().stream().mapToInt(List::size).sum(); + } + summary.add("Samples ("+numSamples+"):"); + summary.add(""); + if(!samples.isEmpty()) { + Map> samps = samples.get(space); + for (String type : samps.keySet()) { + summary.add(type + ": " + samps.get(type).size()); + } + } + summary.add(""); + int numData = 0; + if (datasets.containsKey(space)) { + numData = datasets.get(space).values().stream().mapToInt(List::size).sum(); + } + summary.add("Attached datasets (" + numData + "):"); + summary.add(""); + if (datasets.get(space) != null) { + Map> dsets = datasets.get(space); + for (String dataType : dsets.keySet()) { + summary.add(dataType + ": " + dsets.get(dataType).size()); + } + } + + summary.add(""); + } + + for(String line : summary) { + System.out.println(line); + } + + Path outputPath = Paths.get(Configuration.LOG_PATH.toString(), + "spaces_summary_"+getTimeStamp()+".txt"); + if(outpath!=null) { + outputPath = Paths.get(outpath); + } + SummaryWriter summaryWriter = new FileSystemWriter(outputPath); + try { + summaryWriter.reportSummary(summary); + } catch (IOException e) { + throw new RuntimeException("Could not write summary file."); + } + } + + private String getTimeStamp() { + final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); + return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter).toString(); + } +} diff --git a/src/main/java/life/qbic/model/DatasetWithProperties.java b/src/main/java/life/qbic/model/DatasetWithProperties.java new file mode 100644 index 0000000..d9a1534 --- /dev/null +++ b/src/main/java/life/qbic/model/DatasetWithProperties.java @@ -0,0 +1,60 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSetType; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.person.Person; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +/** + * Wrapper class for openBIS DataSets that collects additional information, e.g. from samples, + * experiments etc. further up in the hierarchy. + */ +public class DatasetWithProperties { + + private final DataSet dataset; + private final Map properties; + + public DatasetWithProperties(DataSet dataset) { + this.dataset = dataset; + this.properties = new HashMap<>(); + } + + public void addProperty(String key, String value) { + this.properties.put(key, value); + } + + public String getProperty(String key) { + return properties.get(key); + } + + public Map getProperties() { + return properties; + } + + public DataSet getDataset() { + return dataset; + } + + public String getCode() { + return dataset.getCode(); + } + + public Experiment getExperiment() { + return dataset.getExperiment(); + } + + public DataSetType getType() { + return dataset.getType(); + } + + public Person getRegistrator() { + return dataset.getRegistrator(); + } + + public Date getRegistrationDate() { + return dataset.getRegistrationDate(); + } +} diff --git a/src/main/java/life/qbic/model/download/FileSystemWriter.java b/src/main/java/life/qbic/model/download/FileSystemWriter.java index 4cf0cb9..59b6419 100644 --- a/src/main/java/life/qbic/model/download/FileSystemWriter.java +++ b/src/main/java/life/qbic/model/download/FileSystemWriter.java @@ -14,7 +14,7 @@ * * @author: Sven Fillinger, Andreas Friedrich */ -public class FileSystemWriter implements ModelReporter { +public class FileSystemWriter implements SummaryWriter { /** * File that stores the summary report content for valid checksums. diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index f5e1654..cf822a7 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -8,6 +8,7 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.search.DataSetSearchCriteria; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.EntityKind; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.search.ExperimentSearchCriteria; @@ -35,6 +36,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import life.qbic.model.SampleTypeConnection; import org.apache.logging.log4j.LogManager; @@ -45,29 +47,6 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); OpenBIS openBIS; - /** - * Constructor for a QBiCDataDownloader instance - * - * @param AppServerUri The openBIS application server URL (AS) - * @param sessionToken The session token for the datastore & application servers - */ - public OpenbisConnector( - String AppServerUri, - String sessionToken) { - /* - this.sessionToken = sessionToken; - - if (!AppServerUri.isEmpty()) { - applicationServer = - HttpInvokerUtils.createServiceStub( - IApplicationServerApi.class, AppServerUri + IApplicationServerApi.SERVICE_URL, 10000); - } else { - applicationServer = null; - } - - */ - } - public OpenbisConnector(OpenBIS authentication) { this.openBIS = authentication; } @@ -124,12 +103,11 @@ public List listDatasetsOfExperiment(List spaces, String experi return openBIS.searchDataSets(criteria, options).getObjects(); } - public void downloadDataset(String targetPath, String datasetID) throws IOException { + public void downloadDataset(String targetPath, String datasetID) { DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetID), ""); - System.err.println(fileToDownload); // Setting recursive flag to true will return both subfolders and files options.setRecursive(true); @@ -137,18 +115,24 @@ public void downloadDataset(String targetPath, String datasetID) throws IOExcept InputStream stream = openBIS.downloadFiles(new ArrayList<>(List.of(fileToDownload)), options); DataSetFileDownloadReader reader = new DataSetFileDownloadReader(stream); - DataSetFileDownload file = null; + DataSetFileDownload file; while ((file = reader.read()) != null) { DataSetFile df = file.getDataSetFile(); String currentPath = df.getPath().replace("original", ""); if (df.isDirectory()) { File newDir = new File(targetPath, currentPath); if (!newDir.exists()) { - newDir.mkdirs(); + if(!newDir.mkdirs()) { + throw new RuntimeException("Could not create folders for downloaded dataset."); + } } } else { File toWrite = new File(targetPath, currentPath); - copyInputStreamToFile(file.getInputStream(), toWrite); + try { + copyInputStreamToFile(file.getInputStream(), toWrite); + } catch (IOException e) { + throw new RuntimeException(e); + } } } } @@ -195,6 +179,165 @@ public Map queryFullSampleHierarchy(List return hierarchy; } + private Optional getPropertyFromSampleHierarchy(String propertyName, List samples) { + for(Sample s : samples) { + if(s.getProperties().containsKey(propertyName)) { + return Optional.of(s.getProperties().get(propertyName)); + } + return getPropertyFromSampleHierarchy(propertyName, s.getParents()); + } + return Optional.empty(); + } + + public Optional findPropertyInSampleHierarchy(String propertyName, + ExperimentIdentifier experimentId) { + return getPropertyFromSampleHierarchy(propertyName, + getSamplesWithAncestorsOfExperiment(experimentId)); + } + + public Map> getExperimentsBySpace(List spaces) { + Map> result = new HashMap<>(); + ExperimentFetchOptions options = new ExperimentFetchOptions(); + options.withProject().withSpace(); + ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); + criteria.withProject().withSpace().withCodes().thatIn(spaces); + for (Experiment e : openBIS.searchExperiments(criteria, options).getObjects()) { + String space = e.getProject().getSpace().getCode(); + if(result.containsKey(space)) { + result.get(space).add(e); + } else { + result.put(space, new ArrayList<>()); + } + } + return result; + } + + public Map> getSamplesBySpace(List spaces) { + Map> result = new HashMap<>(); + SampleFetchOptions options = new SampleFetchOptions(); + options.withSpace(); + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withSpace().withCodes().thatIn(spaces); + for (Sample s : openBIS.searchSamples(criteria, options).getObjects()) { + String space = s.getSpace().getCode(); + if(!result.containsKey(space)) { + result.put(space, new ArrayList<>()); + } + result.get(space).add(s); + } + return result; + } + + public Map>> getExperimentsByTypeAndSpace(List spaces) { + Map>> result = new HashMap<>(); + ExperimentFetchOptions options = new ExperimentFetchOptions(); + options.withProject().withSpace(); + options.withType(); + + ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); + criteria.withProject().withSpace().withCodes().thatIn(spaces); + for (Experiment exp : openBIS.searchExperiments(criteria, options).getObjects()) { + String space = exp.getProject().getSpace().getCode(); + String type = exp.getType().getCode(); + if(!result.containsKey(space)) { + Map> typeMap = new HashMap<>(); + typeMap.put(type, new ArrayList<>(Arrays.asList(exp))); + result.put(space, typeMap); + } else { + Map> typeMap = result.get(space); + if(!typeMap.containsKey(type)) { + typeMap.put(type, new ArrayList<>()); + } + typeMap.get(type).add(exp); + } + } + return result; + } + + public Map>> getSamplesByTypeAndSpace(List spaces) { + Map>> result = new HashMap<>(); + SampleFetchOptions options = new SampleFetchOptions(); + options.withSpace(); + options.withType(); + + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withSpace().withCodes().thatIn(spaces); + for (Sample s : openBIS.searchSamples(criteria, options).getObjects()) { + String space = s.getSpace().getCode(); + String type = s.getType().getCode(); + if(!result.containsKey(space)) { + Map> typeMap = new HashMap<>(); + typeMap.put(type, new ArrayList<>(Arrays.asList(s))); + result.put(space, typeMap); + } else { + Map> typeMap = result.get(space); + if(!typeMap.containsKey(type)) { + typeMap.put(type, new ArrayList<>()); + } + typeMap.get(type).add(s); + } + } + return result; + } + + public Map>> getDatasetsByTypeAndSpace(List spaces) { + Map>> result = new HashMap<>(); + DataSetFetchOptions options = new DataSetFetchOptions(); + options.withSample().withSpace(); + options.withExperiment().withProject().withSpace(); + options.withType(); + DataSetSearchCriteria criteria = new DataSetSearchCriteria(); + criteria.withOrOperator(); + criteria.withSample().withSpace().withCodes().thatIn(spaces); + criteria.withExperiment().withProject().withSpace().withCodes().thatIn(spaces); + for (DataSet d : openBIS.searchDataSets(criteria, options).getObjects()) { + String space = getSpaceFromSampleOrExperiment(d); + String type = d.getType().getCode(); + if(!result.containsKey(space)) { + Map> typeMap = new HashMap<>(); + typeMap.put(type, new ArrayList<>(Arrays.asList(d))); + result.put(space, typeMap); + } else { + Map> typeMap = result.get(space); + if(!typeMap.containsKey(type)) { + typeMap.put(type, new ArrayList<>()); + } + typeMap.get(type).add(d); + } + } + return result; + } + + private String getSpaceFromSampleOrExperiment(DataSet d) { + try { + if (d.getSample() != null) { + return d.getSample().getSpace().getCode(); + } + if (d.getExperiment() != null) { + return d.getExperiment().getProject().getSpace().getCode(); + } + } catch (NullPointerException e) { + + } + System.out.println("Dataset " + d + "does not seem to be attached to a space"); + return "NO SPACE"; + } + + private List getSamplesWithAncestorsOfExperiment(ExperimentIdentifier experimentId) { + SampleFetchOptions allProps = new SampleFetchOptions(); + allProps.withType(); + allProps.withProperties(); + SampleFetchOptions withAncestors = new SampleFetchOptions(); + withAncestors.withParentsUsing(allProps); + withAncestors.withProperties(); + withAncestors.withType(); + + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withExperiment().withId().thatEquals(experimentId); + + return openBIS.searchSamples(criteria, withAncestors).getObjects(); + } + public List findDataSets(List codes) { if (codes.isEmpty()) { return new ArrayList<>(); @@ -202,6 +345,7 @@ public List findDataSets(List codes) { DataSetSearchCriteria criteria = new DataSetSearchCriteria(); criteria.withCodes().thatIn(codes); DataSetFetchOptions options = new DataSetFetchOptions(); + options.withExperiment(); return openBIS.searchDataSets(criteria, options).getObjects(); } @@ -212,4 +356,5 @@ public boolean experimentExists(String experimentID) { return !openBIS.searchExperiments(criteria, new ExperimentFetchOptions()).getObjects() .isEmpty(); } + } diff --git a/src/main/java/life/qbic/model/download/ModelReporter.java b/src/main/java/life/qbic/model/download/SummaryWriter.java similarity index 65% rename from src/main/java/life/qbic/model/download/ModelReporter.java rename to src/main/java/life/qbic/model/download/SummaryWriter.java index 875ffb4..dbf2438 100644 --- a/src/main/java/life/qbic/model/download/ModelReporter.java +++ b/src/main/java/life/qbic/model/download/SummaryWriter.java @@ -1,11 +1,9 @@ package life.qbic.model.download; import java.io.IOException; -import java.net.URL; -import java.nio.file.Path; import java.util.List; -public interface ModelReporter { +public interface SummaryWriter { void reportSummary(List summary) throws IOException; }