diff --git a/misc/sync_dependency.sh b/misc/sync_dependency.sh new file mode 100755 index 00000000000..316bbc39d3b --- /dev/null +++ b/misc/sync_dependency.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +function yellow (){ + echo "$(tput setaf 3)$1$(tput setaf 7)" +} +function green (){ + echo "$(tput setaf 2)$1$(tput setaf 7)" +} +function cyan (){ + echo "$(tput setaf 6)$1$(tput setaf 7)" +} + +function printUsage(){ + echo "" + yellow "Release an OpenCB project." + echo "" + echo "Usage: $(basename $0) --biodata|-b|--java-common-libs|-j" + echo "" + cyan "Options:" + green " -j --java-common-libs STRING Update java-common-libs dependency" + green " -b --biodata STRING Update biodata dependency" + echo "" +} + +## Check if the repo status is clean. +function check_repo_clean() { + GIT_STATUS=$(git status --short) + if [ -n "$GIT_STATUS" ]; then + yellow "Repository is not clean:" + yellow "$GIT_STATUS" + exit + fi +} + +## This function removes TASK-XXX- if exists, otherwise it adds it. +function toggle_version() { + local BRANCH=$1 + if [[ "$POM_DEPENDENCY_VERSION" == *"$BRANCH"* ]]; then + ## Remove TASK-XXX- from the current version + ## Example: remove 'TASK-1234-' from 2.6.0-TASK-1234-SNAPSHOT + NEW_VERSION=${POM_DEPENDENCY_VERSION/"$BRANCH-"} + else + ## Add 'TASK-XXX-' to the current version + ## Example: 2.6.0-SNAPSHOT --> 2.6.0-TASK-1234-SNAPSHOT + CLEAN_RELEASE_VERSION=$(echo "$POM_DEPENDENCY_VERSION" | cut -d "-" -f 1) + TAG_VERSION=$(echo "$POM_DEPENDENCY_VERSION" | cut -d "-" -f 2) + NEW_VERSION="$CLEAN_RELEASE_VERSION-$BRANCH-$TAG_VERSION" + fi +} + +## Change version in the dependency. +## Usage: update_dependency "$DEPENDENCY_REPO" "$NEW_VERSION" "$BRANCH_NAME" +function update_dependency() { + ## Save current directory + local pwd=$PWD + cd "$1" || exit 2 + check_repo_clean + git checkout "$3" + ## Check branch exists + local BRANCH=$(git branch --show-current) + if [ "$BRANCH" != "$3" ]; then + yellow "Branch '$3' does not exist" + exit + fi + ## Rename and commit new version + mvn versions:set -DnewVersion="$2" -DgenerateBackupPoms=false + git commit -am "Update version to $2" + ## Restore directory + cd "$pwd" || exit 2 +} + +## At least one parameter is required. +if [ -z "$1" ]; then + printUsage + exit 1 +fi + +while [[ $# -gt 0 ]]; do + key="$1" + if [ -n "$2" ]; then + DEPENDENCY_REPO="$2" + fi + case $key in + -h | --help) + printUsage + exit 0 + ;; + -j | --java-common-libs) + LIB="JAVA_COMMONS_LIB" + if [ -z "$DEPENDENCY_REPO" ]; then + DEPENDENCY_REPO="../java-common-libs" + else + shift + fi + shift # past argument + ;; + -b | --biodata) + LIB="BIODATA" + if [ -z "$DEPENDENCY_REPO" ]; then + DEPENDENCY_REPO="../biodata" + else + shift + fi + shift # past argument + ;; + *) # unknown option + echo "Unknown option $key" + printUsage + exit 1 + ;; + esac +done + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +CURRENT_DIR=$PWD +cd "$SCRIPT_DIR" || exit 2 +cd .. +BRANCH_NAME=$(git branch --show-current) +if [[ "$BRANCH_NAME" == "TASK-"* ]]; then + check_repo_clean "$BRANCH_NAME" +else + yellow "[$BRANCH_NAME] The branch name must start with TASK-" + yellow "$GIT_STATUS" + exit +fi + +function update_library(){ + local LIBRARY="$1" + POM_DEPENDENCY_VERSION=$(grep -m 1 "$LIBRARY" pom.xml | cut -d ">" -f 2 | cut -d "<" -f 1) + toggle_version "$BRANCH_NAME" + update_dependency "$DEPENDENCY_REPO" "$NEW_VERSION" "$BRANCH_NAME" + mvn versions:set-property -Dproperty=java-common-libs.version -DnewVersion="$NEW_VERSION" -DgenerateBackupPoms=false + git commit -am "Update '$LIBRARY' dependency to $NEW_VERSION" +} + + +if [ "$LIB" = "JAVA_COMMONS_LIB" ];then + update_library java-common-libs.version +fi +if [ "$LIB" = "BIODATA" ];then + update_library biodata.version +fi + +yellow "The new dependency version is $NEW_VERSION" +cd "$CURRENT_DIR" || exit 2 diff --git a/opencga b/opencga deleted file mode 100644 index 97288a06c67..00000000000 --- a/opencga +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - - -

OpenCGA server

-
- - - \ No newline at end of file diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml index 90cb0d4375e..5d6e9c1dbd7 100644 --- a/opencga-analysis/pom.xml +++ b/opencga-analysis/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/AnalysisUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/AnalysisUtils.java index 88d38950f49..628bc63b2ff 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/AnalysisUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/AnalysisUtils.java @@ -6,8 +6,11 @@ import org.opencb.opencga.catalog.db.api.FileDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.managers.FileManager; +import org.opencb.opencga.catalog.managers.JobManager; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.response.OpenCGAResult; import java.io.*; @@ -121,4 +124,54 @@ public static Map> parseRelatednessThresholds(Path th } return thresholds; } + + public static boolean waitFor(String jobId, String study, JobManager jobManager, String token) throws ToolException, CatalogException { + Query query = new Query("id", jobId); + OpenCGAResult result = jobManager.search(study, query, QueryOptions.empty(), token); + Job job = result.first(); + String status = job.getInternal().getStatus().getId(); + + while (status.equals(Enums.ExecutionStatus.PENDING) || status.equals(Enums.ExecutionStatus.RUNNING) + || status.equals(Enums.ExecutionStatus.QUEUED) || status.equals(Enums.ExecutionStatus.READY) + || status.equals(Enums.ExecutionStatus.REGISTERING)) { + try { + // Sleep for 30 seconds + Thread.sleep(30000); + result = jobManager.search(study, query, QueryOptions.empty(), token); + job = result.first(); + } catch (CatalogException | InterruptedException e) { + new ToolException("Error waiting for job '" + jobId + "': " + e.getMessage()); + } + status = job.getInternal().getStatus().getId(); + } + return status.equals(Enums.ExecutionStatus.DONE) ? true : false; + } + + public static Job getJob(String jobId, String study, JobManager jobManager, String token) throws ToolException, CatalogException { + Query query = new Query("id", jobId); + OpenCGAResult result = jobManager.search(study, query, QueryOptions.empty(), token); + Job job = result.first(); + if (job == null) { + new ToolException("Error getting job '" + jobId + "' from study '" + study + "'."); + } + return job; + } + + public static final String JOBS_IN_JOBDIR = "JOBS"; + + public static String getJobBaseDir(String path) { + int index = path.indexOf(JOBS_IN_JOBDIR); + if (index == -1) { + return null; + } + return path.substring(0, index + 5); + } + + public static String getJobFileRelativePath(String path) { + int index = path.indexOf(JOBS_IN_JOBDIR); + if (index == -1) { + return null; + } + return path.substring(index + 5); + } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java index 5285e538a26..47b8b3d0939 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java @@ -228,6 +228,7 @@ public OpenCGAResult coverageStats(String studyIdStr, String } String species = projectQueryResult.first().getOrganism().getScientificName(); String assembly = projectQueryResult.first().getOrganism().getAssembly(); + String dataRelease = projectQueryResult.first().getCellbase().getDataRelease(); for (String geneName : geneNames) { @@ -248,9 +249,9 @@ public OpenCGAResult coverageStats(String studyIdStr, String // Query CellBase to get gene coordinates and then apply the offset (up and downstream) to create a gene region - CellBaseClient cellBaseClient = new CellBaseClient(storageEngineFactory.getVariantStorageEngine().getConfiguration().getCellbase() + CellBaseClient cellBaseClient = new CellBaseClient(species, assembly, dataRelease, projectQueryResult.first().getCellbase() .toClientConfiguration()); - GeneClient geneClient = new GeneClient(species, assembly, cellBaseClient.getClientConfiguration()); + GeneClient geneClient = cellBaseClient.getGeneClient(); Gene gene = geneClient.get(Collections.singletonList(geneName), QueryOptions.empty()).firstResult(); if (gene != null) { List transcriptCoverageStatsList = new ArrayList<>(); @@ -445,9 +446,10 @@ public List mergeRegions(List regions, List genes, boole // Query CellBase to get gene coordinates and then apply the offset (up and downstream) to create a gene region String species = projectQueryResult.first().getOrganism().getScientificName(); String assembly = projectQueryResult.first().getOrganism().getAssembly(); - CellBaseClient cellBaseClient = new CellBaseClient(storageEngineFactory.getVariantStorageEngine().getConfiguration().getCellbase() + String dataRelease = projectQueryResult.first().getCellbase().getDataRelease(); + CellBaseClient cellBaseClient = new CellBaseClient(species, assembly, dataRelease, projectQueryResult.first().getCellbase() .toClientConfiguration()); - GeneClient geneClient = new GeneClient(species, assembly, cellBaseClient.getClientConfiguration()); + GeneClient geneClient = cellBaseClient.getGeneClient(); List response = geneClient.get(genes, QueryOptions.empty()).allResults(); if (CollectionUtils.isNotEmpty(response)) { for (Gene gene : response) { @@ -500,15 +502,13 @@ private void updateRegionMap(Region region, Map map) { // PRIVATE METHODS //------------------------------------------------------------------------- - public Map> getExonRegionsPerTranscript(String geneName, String species, String assembly) + public Map> getExonRegionsPerTranscript(String geneName, CellBaseClient cellBaseClient) throws StorageEngineException, IOException { // Init region map, where key = transcript and value = list of exon regions Map> regionMap = new HashMap<>(); // Query CellBase to get gene coordinates and then apply the offset (up and downstream) to create a gene region - CellBaseClient cellBaseClient = new CellBaseClient(storageEngineFactory.getVariantStorageEngine().getConfiguration().getCellbase() - .toClientConfiguration()); - GeneClient geneClient = new GeneClient(species, assembly, cellBaseClient.getClientConfiguration()); + GeneClient geneClient = cellBaseClient.getGeneClient(); Gene gene = geneClient.get(Collections.singletonList(geneName), QueryOptions.empty()).firstResult(); if (gene != null) { // Create region from gene coordinates diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java index 43af49f934c..d4d08bba37f 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java @@ -61,7 +61,6 @@ public class AlignmentQcAnalysis extends OpenCgaToolScopeStudy { private boolean runStats = true; private boolean runFlagStats = true; private boolean runFastqc = true; - private boolean runHsmetrics = true; @Override protected void check() throws Exception { @@ -99,33 +98,10 @@ protected void check() throws Exception { (!analysisParams.isOverwrite() && alignmentQc != null && alignmentQc.getFastQcMetrics() != null)) { runFastqc = false; } - if (skipValues.contains(AlignmentQcParams.HS_METRICS_SKIP_VALUE) - || - (!analysisParams.isOverwrite() && alignmentQc != null && alignmentQc.getHsMetrics() != null) - || - StringUtils.isEmpty(analysisParams.getBedFile()) - || - StringUtils.isEmpty(analysisParams.getDictFile())) { - runHsmetrics = false; - } } } catch (CatalogException e) { throw new ToolException("Error accessing to the BAM file '" + analysisParams.getBamFile() + "'", e); } - - if (runHsmetrics) { - try { - AnalysisUtils.getCatalogFile(analysisParams.getBedFile(), study, catalogManager.getFileManager(), token); - } catch (CatalogException e) { - throw new ToolException("Error accessing to the BED file '" + analysisParams.getBedFile() + "'", e); - } - - try { - AnalysisUtils.getCatalogFile(analysisParams.getDictFile(), study, catalogManager.getFileManager(), token); - } catch (CatalogException e) { - throw new ToolException("Error accessing to the dictionary file '" + analysisParams.getDictFile() + "'", e); - } - } } @Override @@ -136,7 +112,6 @@ protected void run() throws ToolException { String statsJobId = null; String flagStatsJobId = null; String fastQcMetricsJobId = null; - String hsMetricsJobId = null; try { if (runFlagStats) { @@ -189,29 +164,10 @@ protected void run() throws ToolException { addWarning("Error launching job for Alignment FastQC Metrics Analysis: " + e.getMessage()); } - try { - if (runHsmetrics) { - // HS metrics - params = new AlignmentHsMetricsParams(analysisParams.getBamFile(), analysisParams.getBedFile(), - analysisParams.getDictFile(), null).toParams(new ObjectMap(ParamConstants.STUDY_PARAM, study)); - - OpenCGAResult hsMetricsJobResult = catalogManager.getJobManager() - .submit(study, AlignmentHsMetricsAnalysis.ID, Enums.Priority.MEDIUM, params, null, - "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), - token); - hsMetricsJobId = hsMetricsJobResult.first().getId(); - addEvent(Event.Type.INFO, "Submit job " + hsMetricsJobId + " to compute HS metrics (" + AlignmentHsMetricsAnalysis.ID - + ")"); - } - } catch (CatalogException e) { - addWarning("Error launching job for Alignment HS Metrics Analysis: " + e.getMessage()); - } - // Wait for those jobs before saving QC SamtoolsFlagstats samtoolsFlagstats = null; SamtoolsStats samtoolsStats = null; FastQcMetrics fastQcMetrics = null; - HsMetrics hsMetrics = null; if (flagStatsJobId != null) { try { @@ -249,17 +205,6 @@ protected void run() throws ToolException { addWarning("Error waiting for job '" + fastQcMetricsJobId + "' (Alignment FastQC Metrics Analysis): " + e.getMessage()); } } - if (hsMetricsJobId != null) { - try { - if (waitFor(hsMetricsJobId)) { - Job job = getJob(hsMetricsJobId); - logger.info("Alignment HS Metrics Analysis, job.outDir = " + job.getOutDir()); - hsMetrics = AlignmentHsMetricsAnalysis.parseResults(Paths.get(job.getOutDir().getUri().getPath())); - } - } catch (Exception e) { - addWarning("Error waiting for job '" + hsMetricsJobId + "' (Alignment FastQC Metrics Analysis): " + e.getMessage()); - } - } // Update quality control for the catalog file catalogBamFile = AnalysisUtils.getCatalogFile(analysisParams.getBamFile(), study, catalogManager.getFileManager(), token); @@ -284,10 +229,6 @@ protected void run() throws ToolException { qc.getAlignment().setFastQcMetrics(fastQcMetrics); saveQc = true; } - if (hsMetrics != null) { - qc.getAlignment().setHsMetrics(hsMetrics); - saveQc = true; - } if (saveQc) { catalogManager.getFileManager().update(getStudy(), catalogBamFile.getId(), new FileUpdateParams().setQualityControl(qc), @@ -324,7 +265,7 @@ private boolean waitFor(String jobId) throws ToolException { return status.equals(Enums.ExecutionStatus.DONE) ? true : false; } - private Job getJob(String jobId) throws ToolException { + private Job getJob(String jobId) { Job job = null; try { Query query = new Query("id", jobId); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java index a5ce5a6cf96..2e9f8a022aa 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java @@ -35,7 +35,6 @@ import org.opencb.biodata.tools.clinical.ClinicalVariantCreator; import org.opencb.biodata.tools.clinical.DefaultClinicalVariantCreator; import org.opencb.biodata.tools.pedigree.ModeOfInheritance; -import org.opencb.cellbase.client.rest.CellBaseClient; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.Query; @@ -98,7 +97,6 @@ public class ClinicalInterpretationManager extends StorageManager { private ClinicalVariantEngine clinicalVariantEngine; private VariantStorageManager variantStorageManager; - protected CellBaseClient cellBaseClient; protected AlignmentStorageManager alignmentStorageManager; private VariantCatalogQueryUtils catalogQueryUtils; @@ -138,7 +136,6 @@ public ClinicalInterpretationManager(CatalogManager catalogManager, StorageEngin this.clinicalAnalysisManager = catalogManager.getClinicalAnalysisManager(); this.variantStorageManager = new VariantStorageManager(catalogManager, StorageEngineFactory.get(storageConfiguration)); - this.cellBaseClient = new CellBaseClient(storageConfiguration.getCellbase().toClientConfiguration()); this.alignmentStorageManager = new AlignmentStorageManager(catalogManager, StorageEngineFactory.get(storageConfiguration)); this.catalogQueryUtils = new VariantCatalogQueryUtils(catalogManager); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java index f55b4ca6bb0..a103568d020 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java @@ -18,11 +18,19 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.opencb.biodata.models.clinical.qc.SampleQcVariantStats; +import org.opencb.biodata.formats.alignment.picard.HsMetrics; +import org.opencb.biodata.formats.alignment.samtools.SamtoolsFlagstats; +import org.opencb.biodata.formats.alignment.samtools.SamtoolsStats; +import org.opencb.biodata.formats.sequence.fastqc.FastQcMetrics; +import org.opencb.biodata.models.clinical.qc.*; import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.AnalysisUtils; +import org.opencb.opencga.analysis.alignment.qc.AlignmentFastQcMetricsAnalysis; +import org.opencb.opencga.analysis.alignment.qc.AlignmentFlagStatsAnalysis; +import org.opencb.opencga.analysis.alignment.qc.AlignmentHsMetricsAnalysis; +import org.opencb.opencga.analysis.alignment.qc.AlignmentStatsAnalysis; import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.analysis.variant.genomePlot.GenomePlotAnalysis; @@ -32,10 +40,17 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.alignment.AlignmentFileQualityControl; +import org.opencb.opencga.core.models.alignment.AlignmentQcParams; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.file.FileQualityControl; +import org.opencb.opencga.core.models.file.FileUpdateParams; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.sample.SampleQualityControl; +import org.opencb.opencga.core.models.sample.SampleUpdateParams; +import org.opencb.opencga.core.models.sample.SampleVariantQualityControlMetrics; import org.opencb.opencga.core.models.variant.GenomePlotAnalysisParams; import org.opencb.opencga.core.models.variant.MutationalSignatureAnalysisParams; import org.opencb.opencga.core.models.variant.SampleQcAnalysisParams; @@ -46,9 +61,9 @@ import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; -import java.util.Map; +import java.util.*; +import static org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis.*; import static org.opencb.opencga.core.models.study.StudyPermissions.Permissions.WRITE_SAMPLES; @Tool(id = SampleQcAnalysis.ID, resource = Enums.Resource.SAMPLE, description = SampleQcAnalysis.DESCRIPTION) @@ -64,7 +79,8 @@ public class SampleQcAnalysis extends OpenCgaToolScopeStudy { private Path genomePlotConfigPath; private boolean runVariantStats = true; - private boolean runSignature = true; + private boolean runSignatureCatalogue = true; + private boolean runSignatureFitting = true; private boolean runGenomePlot = true; @Override @@ -94,38 +110,51 @@ protected void check() throws Exception { throw new ToolException("Sample '" + analysisParams.getSample() + "' not found."); } - String msg; - - // Check variant stats - final String OPENCGA_ALL = "ALL"; - if (OPENCGA_ALL.equals(analysisParams.getVsId())) { - msg = "Invalid parameters: " + OPENCGA_ALL + " is a reserved word, you can not use as a variant stats ID"; - addWarning(msg); - logger.warn(msg); - runVariantStats = false; - } - - if (StringUtils.isEmpty(analysisParams.getVsId()) && analysisParams.getVsQuery() != null - && !analysisParams.getVsQuery().toParams().isEmpty()) { - msg = "Invalid parameters: if variant stats ID is empty, variant stats query must be empty"; - addWarning(msg); - logger.warn(msg); - runVariantStats = false; - } - if (StringUtils.isNotEmpty(analysisParams.getVsId()) - && (analysisParams.getVsQuery() == null || analysisParams.getVsQuery().toParams().isEmpty())) { - msg = "Invalid parameters: if you provide a variant stats ID, variant stats query can not be empty"; - addWarning(msg); - logger.warn(msg); - runVariantStats = false; + // Prepare flags + String skip = null; + if (StringUtils.isNotEmpty(analysisParams.getSkip())) { + skip = analysisParams.getSkip().toLowerCase().replace(" ", ""); } - if (StringUtils.isEmpty(analysisParams.getVsId())) { - analysisParams.setVsId(OPENCGA_ALL); + if (StringUtils.isNotEmpty(skip)) { + Set skipValues = new HashSet<>(Arrays.asList(skip.split(","))); + if (skipValues.contains(SampleQcAnalysisParams.VARIANT_STATS_SKIP_VALUE)) { + runVariantStats = false; + } + if (skipValues.contains(SampleQcAnalysisParams.SIGNATURE_SKIP_VALUE) + || skipValues.contains(SampleQcAnalysisParams.SIGNATURE_CATALOGUE_SKIP_VALUE)) { + runSignatureCatalogue = false; + } + if (skipValues.contains(SampleQcAnalysisParams.SIGNATURE_SKIP_VALUE) + || skipValues.contains(SampleQcAnalysisParams.SIGNATURE_FITTING_SKIP_VALUE)) { + runSignatureFitting = false; + } + if (skipValues.contains(SampleQcAnalysisParams.GENOME_PLOT_SKIP_VALUE)) { + runGenomePlot = false; + } } - if (analysisParams.getVsQuery() == null) { - runVariantStats = false; - } else { + // Check variant stats + if (runVariantStats) { + final String OPENCGA_ALL = "ALL"; + if (OPENCGA_ALL.equals(analysisParams.getVsId())) { + new ToolException("Invalid parameters: " + OPENCGA_ALL + " is a reserved word, you can not use as a variant stats ID"); + } + + if (StringUtils.isEmpty(analysisParams.getVsId()) && analysisParams.getVsQuery() != null + && !analysisParams.getVsQuery().toParams().isEmpty()) { + new ToolException("Invalid parameters: if variant stats ID is empty, variant stats query must be empty"); + } + if (StringUtils.isNotEmpty(analysisParams.getVsId()) + && (analysisParams.getVsQuery() == null || analysisParams.getVsQuery().toParams().isEmpty())) { + new ToolException("Invalid parameters: if you provide a variant stats ID, variant stats query can not be empty"); + } + if (StringUtils.isEmpty(analysisParams.getVsId())) { + analysisParams.setVsId(OPENCGA_ALL); + } + + if (analysisParams.getVsQuery() == null) { + new ToolException("Invalid parameters: variant stats query is empty"); + } if (sample.getQualityControl() != null && sample.getQualityControl().getVariant() != null) { if (CollectionUtils.isNotEmpty(sample.getQualityControl().getVariant().getVariantStats()) && OPENCGA_ALL.equals(analysisParams.getVsId())) { @@ -133,10 +162,8 @@ protected void check() throws Exception { } else { for (SampleQcVariantStats variantStats : sample.getQualityControl().getVariant().getVariantStats()) { if (variantStats.getId().equals(analysisParams.getVsId())) { - msg = "Invalid parameters: variant stats ID '" + analysisParams.getVsId() + "' is already used"; - addWarning(msg); - logger.warn(msg); - runVariantStats = false; + throw new ToolException("Invalid parameters: variant stats ID '" + analysisParams.getVsId() + + "' is already used"); } } } @@ -144,23 +171,33 @@ protected void check() throws Exception { } // Check mutational signature - if (StringUtils.isEmpty(analysisParams.getMsQuery())) { - runSignature = false; + if (runSignatureCatalogue) { + if (StringUtils.isEmpty(analysisParams.getMsQuery())) { + new ToolException("Invalid parameters: mutational signature query is empty"); + } } - if (runSignature && !sample.isSomatic()) { - msg = "Skipping mutational signature: sample '" + sample.getId() + "' is not somatic."; + if (runSignatureCatalogue && !sample.isSomatic()) { + String msg = "Skipping mutational signature catalog analysis: sample '" + sample.getId() + "' is not somatic."; addWarning(msg); logger.warn(msg); - runSignature = false; + runSignatureCatalogue = false; + } + + if (runSignatureFitting && !sample.isSomatic()) { + String msg = "Skipping mutational signature fitting analysis: sample '" + sample.getId() + "' is not somatic."; + addWarning(msg); + logger.warn(msg); + runSignatureFitting = false; } // Check genome plot - if (StringUtils.isEmpty(analysisParams.getGpConfigFile())) { - runGenomePlot = false; - } else { + if (runGenomePlot) { + if (StringUtils.isEmpty(analysisParams.getGpConfigFile())) { + new ToolException("Invalid parameters: genome plot configuration file is empty"); + } if (runGenomePlot && !sample.isSomatic()) { - msg = "Skipping genome plot: sample '" + sample.getId() + "' is not somatic."; + String msg = "Skipping genome plot: sample '" + sample.getId() + "' is not somatic."; addWarning(msg); logger.warn(msg); runGenomePlot = false; @@ -169,25 +206,21 @@ protected void check() throws Exception { catalogManager.getFileManager(), getToken()); genomePlotConfigPath = Paths.get(genomePlotConfFile.getUri().getPath()); if (!genomePlotConfigPath.toFile().exists()) { - msg = "Invalid parameters: genome plot configuration file does not exist (" + genomePlotConfigPath + ")"; - addWarning(msg); - logger.warn(msg); - runGenomePlot = false; + new ToolException("Invalid parameters: genome plot configuration file does not exist (" + genomePlotConfigPath + ")"); } } } - } @Override protected void run() throws ToolException { step(() -> { - try { - Map params; - OpenCGAResult variantStatsJobResult; - OpenCGAResult signatureJobResult; - OpenCGAResult genomePlotJobResult; + Map params; + String variantStatsJobId = null; + String signatureJobId = null; + String genomePlotJobId = null; + try { if (runVariantStats) { // Run variant stats params = new SampleVariantStatsAnalysisParams(Collections.singletonList(analysisParams.getSample()), null, null, true, @@ -195,64 +228,144 @@ protected void run() throws ToolException { analysisParams.getVsQuery()) .toParams(new ObjectMap(ParamConstants.STUDY_PARAM, getStudy())); - variantStatsJobResult = catalogManager.getJobManager() - .submit(getStudy(), SampleVariantStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " + OpenCGAResult variantStatsJobResult = catalogManager.getJobManager() + .submit(study, SampleVariantStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), token); - addEvent(Event.Type.INFO, "Submit job " + variantStatsJobResult.first().getId() + " to compute sample variant stats (" - + SampleVariantStatsAnalysis.ID + ")"); + variantStatsJobId = variantStatsJobResult.first().getId(); + addEvent(Event.Type.INFO, "Submit job " + variantStatsJobId + " to compute stats (" + SampleVariantStatsAnalysis.ID + + ")"); } + } catch (CatalogException e) { + addWarning("Error launching job for sample variant stats analysis: " + e.getMessage()); + variantStatsJobId = null; + } - if (runSignature) { + try { + if (runSignatureCatalogue || runSignatureFitting) { // Run mutational signature + logger.info("Preparing to submit the mutational signature analysis job"); - // Be sure to update sample quality control - ObjectMap query = JacksonUtils.getDefaultObjectMapper().readValue(analysisParams.getMsQuery(), ObjectMap.class); - query.append(MutationalSignatureAnalysis.QC_UPDATE_KEYNAME, true); - String queryString = query.toJson(); + String skip = null; + if (!runSignatureCatalogue) { + skip = MutationalSignatureAnalysisParams.SIGNATURE_CATALOGUE_SKIP_VALUE; + } else if (!runSignatureFitting) { + skip = MutationalSignatureAnalysisParams.SIGNATURE_FITTING_SKIP_VALUE; + } params = new MutationalSignatureAnalysisParams() .setId(analysisParams.getMsId()) .setDescription(analysisParams.getMsDescription()) - .setQuery(queryString) + .setSample(analysisParams.getSample()) + .setQuery(analysisParams.getMsQuery()) + .setFitId(analysisParams.getMsFitId()) .setFitMethod(analysisParams.getMsFitMethod()) - .setSigVersion(analysisParams.getMsSigVersion()) - .setOrgan(analysisParams.getMsOrgan()) - .setnBoot(analysisParams.getMsNBoot()) - .setThresholdPerc(analysisParams.getMsThresholdPerc()) - .setThresholdPval(analysisParams.getMsThresholdPval()) - .setMaxRareSigs(analysisParams.getMsMaxRareSigs()) - .setSignaturesFile(analysisParams.getMsSignaturesFile()) - .setRareSignaturesFile(analysisParams.getMsRareSignaturesFile()) + .setFitSigVersion(analysisParams.getMsFitSigVersion()) + .setFitOrgan(analysisParams.getMsFitOrgan()) + .setFitNBoot(analysisParams.getMsFitNBoot()) + .setFitThresholdPerc(analysisParams.getMsFitThresholdPerc()) + .setFitThresholdPval(analysisParams.getMsFitThresholdPval()) + .setFitMaxRareSigs(analysisParams.getMsFitMaxRareSigs()) + .setFitSignaturesFile(analysisParams.getMsFitSignaturesFile()) + .setFitRareSignaturesFile(analysisParams.getMsFitRareSignaturesFile()) + .setSkip(skip) .toParams(new ObjectMap(ParamConstants.STUDY_PARAM, getStudy())); - signatureJobResult = catalogManager.getJobManager() + OpenCGAResult signatureJobResult = catalogManager.getJobManager() .submit(getStudy(), MutationalSignatureAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), token); - addEvent(Event.Type.INFO, "Submit job " + signatureJobResult.first().getId() + " to compute the mutational signature (" + signatureJobId = signatureJobResult.first().getId(); + logger.info("Submitted job {} to compute the mutational signature analysis {}", signatureJobId, + MutationalSignatureAnalysis.ID); + addEvent(Event.Type.INFO, "Submit job " + signatureJobId + " to compute the mutational signature (" + MutationalSignatureAnalysis.ID + ")"); } + } catch (CatalogException e) { + throw new ToolException(e); + } + + try { if (runGenomePlot) { // Run genome plot params = new GenomePlotAnalysisParams(analysisParams.getSample(), analysisParams.getGpId(), analysisParams.getGpDescription(), analysisParams.getGpConfigFile(), null) .toParams(new ObjectMap(ParamConstants.STUDY_PARAM, getStudy())); - genomePlotJobResult = catalogManager.getJobManager() + OpenCGAResult genomePlotJobResult = catalogManager.getJobManager() .submit(getStudy(), GenomePlotAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), token); - addEvent(Event.Type.INFO, "Submit job " + genomePlotJobResult.first().getId() + " to compute genome plot (" - + GenomePlotAnalysis.ID + ")"); + genomePlotJobId = genomePlotJobResult.first().getId(); + addEvent(Event.Type.INFO, "Submit job " + genomePlotJobId + " to compute genome plot (" + GenomePlotAnalysis.ID + + ")"); } + } catch (CatalogException e) { + addWarning("Error launching job for sample genome plot analysis: " + e.getMessage()); + genomePlotJobId = null; + } - // Wait for those jobs ??? -// waitFor(variantStatsJobResult.first().getId()); -// waitFor(signatureJobResult.first().getId()); -// waitFor(genomePlotJobResult.first().getId()); - } catch (CatalogException e) { - throw new ToolException(e); + // Wait for those jobs before saving QC + GenomePlot genomePlot = null; + + if (variantStatsJobId != null) { + try { + logger.info("Waiting for variant stats job: {} ...", variantStatsJobId); + AnalysisUtils.waitFor(variantStatsJobId, getStudy(), catalogManager.getJobManager(), getToken()); + // Sample quality control is updated in the variant stats analysis, nothing more to do here + } catch (Exception e) { + addWarning("Error waiting for job '" + variantStatsJobId + "' (sample variant stats): " + e.getMessage()); + } + } + + if (signatureJobId != null) { + try { + logger.info("Waiting for mutational signature job: {} ...", signatureJobId); + AnalysisUtils.waitFor(signatureJobId, getStudy(), catalogManager.getJobManager(), getToken()); + } catch (Exception e) { + addWarning("Error waiting for job '" + signatureJobId + "' (mutational signature analysis): " + e.getMessage()); + } + } + + if (genomePlotJobId != null) { + try { + if (AnalysisUtils.waitFor(genomePlotJobId, getStudy(), catalogManager.getJobManager(), getToken())) { + Job job = AnalysisUtils.getJob(genomePlotJobId, getStudy(), catalogManager.getJobManager(), getToken()); + + // Parse configuration file + GenomePlotConfig plotConfig = JacksonUtils.getDefaultObjectMapper().readerFor(GenomePlotConfig.class) + .readValue(genomePlotConfigPath.toFile()); + + // Parse genome plot results + genomePlot = GenomePlotAnalysis.parseResults(Paths.get(job.getOutDir().getUri().getPath()), + analysisParams.getGpDescription(), plotConfig); + } + } catch (Exception e) { + addWarning("Error waiting for job '" + genomePlotJobId + "' (genome plot analysis): " + e.getMessage()); + } + } + + // Update quality control for the sample + logger.info("Preparing to save quality control for sample {}", analysisParams.getSample()); + Sample sample = IndividualQcUtils.getValidSampleById(getStudy(), analysisParams.getSample(), catalogManager, token); + if (sample == null) { + throw new ToolException("Can not access to the sample " + analysisParams.getSample() + " in order to save quality control"); + } + SampleQualityControl qc = sample.getQualityControl(); + + // Sanity check + if (qc == null) { + qc = new SampleQualityControl(); + } else if (qc.getVariant() == null) { + qc.setVariant(new SampleVariantQualityControlMetrics()); + } + + if (genomePlot != null) { + qc.getVariant().setGenomePlot(genomePlot); + + catalogManager.getSampleManager().update(getStudy(), sample.getId(), new SampleUpdateParams().setQualityControl(qc), + QueryOptions.empty(), getToken()); + logger.info("Quality control saved for sample {}", sample.getId()); } }); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotAnalysis.java index 85332389073..f7f63f37706 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/genomePlot/GenomePlotAnalysis.java @@ -35,6 +35,8 @@ import org.opencb.opencga.core.tools.annotations.ToolParams; import org.opencb.opencga.core.tools.variant.GenomePlotAnalysisExecutor; +import java.io.IOException; +import java.nio.file.Path; import java.nio.file.Paths; @Tool(id = GenomePlotAnalysis.ID, resource = Enums.Resource.VARIANT) @@ -113,6 +115,18 @@ protected void run() throws ToolException { }); } + public static GenomePlot parseResults(Path outDir, String description, GenomePlotConfig plotConfig) throws IOException { + // Get image file + for (java.io.File imgFile : outDir.toFile().listFiles()) { + if (imgFile.getName().endsWith(GenomePlotAnalysis.SUFFIX_FILENAME)) { + int index = imgFile.getAbsolutePath().indexOf("JOBS/"); + String relativeFilePath = (index == -1 ? imgFile.getName() : imgFile.getAbsolutePath().substring(index)); + return new GenomePlot("", description, plotConfig, relativeFilePath); + } + } + return null; + } + public String getStudy() { return study; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectAnalysis.java new file mode 100644 index 00000000000..23d04db3b65 --- /dev/null +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectAnalysis.java @@ -0,0 +1,341 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.analysis.variant.hrdetect; + +import com.mongodb.client.ListCollectionsIterable; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.biodata.models.clinical.qc.HRDetect; +import org.opencb.biodata.models.clinical.qc.Signature; +import org.opencb.biodata.models.clinical.qc.SignatureFitting; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.AnalysisUtils; +import org.opencb.opencga.analysis.ResourceUtils; +import org.opencb.opencga.analysis.individual.qc.IndividualQcUtils; +import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.sample.SampleQualityControl; +import org.opencb.opencga.core.models.sample.SampleUpdateParams; +import org.opencb.opencga.core.models.sample.SampleVariantQualityControlMetrics; +import org.opencb.opencga.core.models.variant.HRDetectAnalysisParams; +import org.opencb.opencga.core.models.variant.MutationalSignatureAnalysisParams; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.tools.annotations.Tool; +import org.opencb.opencga.core.tools.annotations.ToolParams; +import org.opencb.opencga.core.tools.variant.HRDetectAnalysisExecutor; +import org.opencb.opencga.core.tools.variant.MutationalSignatureAnalysisExecutor; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +@Tool(id = HRDetectAnalysis.ID, resource = Enums.Resource.VARIANT) +public class HRDetectAnalysis extends OpenCgaToolScopeStudy { + + public static final String ID = "hr-detect"; + public static final String DESCRIPTION = "Run HRDetect analysis for a given somatic sample."; + + public final static String HRDETECT_SCORES_FILENAME_DEFAULT = "data_matrix.tsv"; + + @ToolParams + private HRDetectAnalysisParams hrdetectParams = new HRDetectAnalysisParams(); + + private Sample somaticSample; + private Sample germlineSample; + private String assembly; + private Path pathSnvFittingRData; + private Path pathSvFittingRData; + private ObjectMap cnvQuery; + private ObjectMap indelQuery; + + @Override + protected void check() throws Exception { + super.check(); + setUpStorageEngineExecutor(study); + + if (StringUtils.isEmpty(getStudy())) { + throw new ToolException("Missing study"); + } + + assembly = ResourceUtils.getAssembly(catalogManager, study, token); + if (StringUtils.isEmpty(assembly)) { + throw new ToolException("Missing assembly for study '" + study + "'"); + } + + if (StringUtils.isEmpty(hrdetectParams.getSampleId())) { + throw new ToolException("Missing sample ID"); + } + + if (StringUtils.isEmpty(hrdetectParams.getSnvFittingId())) { + throw new ToolException("Missing mutational signature fitting ID for SNV"); + } + + if (StringUtils.isEmpty(hrdetectParams.getSvFittingId())) { + throw new ToolException("Missing mutational signature fitting ID for SV"); + } + + if (StringUtils.isEmpty(hrdetectParams.getCnvQuery())) { + throw new ToolException("Missing CNV query"); + } + + if (StringUtils.isEmpty(hrdetectParams.getIndelQuery())) { + throw new ToolException("Missing INDEL query"); + } + + // Check sample + somaticSample = checkSample(hrdetectParams.getSampleId()); + if (!somaticSample.isSomatic()) { + throw new ToolException("Mismatch sample from CNV query '" + somaticSample.getId() + "' must be somatic"); + } + checkSampleQualityControl(somaticSample); + + SignatureFitting snvFitting = null; + SignatureFitting svFitting = null; + List signatures = somaticSample.getQualityControl().getVariant().getSignatures(); + for (Signature signature : signatures) { + if (CollectionUtils.isNotEmpty(signature.getFittings())) { + for (SignatureFitting fitting : signature.getFittings()) { + if (hrdetectParams.getSnvFittingId().equals(fitting.getId()) && snvFitting == null) { + // Take the first SNV fitting matching ID + snvFitting = fitting; + } else if (hrdetectParams.getSvFittingId().equals(fitting.getId()) && svFitting == null) { + // Take the first SV fitting matching ID + svFitting = fitting; + } + } + } + } + + if (snvFitting == null) { + throw new ToolException("Unable to compute HRDetect analysis. No SNV fitting with ID '" + hrdetectParams.getSnvFittingId() + + "' found for sample '" + hrdetectParams.getSampleId() + "'"); + } + if (svFitting == null) { + throw new ToolException("Unable to compute HRDetect analysis. No SV fitting with ID '" + hrdetectParams.getSvFittingId() + + "' found for sample '" + hrdetectParams.getSampleId() + "'"); + } + + pathSnvFittingRData = getFittingRDataFile(snvFitting.getFiles()); + if (!pathSnvFittingRData.toFile().exists()) { + throw new ToolException("Unable to compute HRDetect analysis. No .rData file found for SNV fitting '" + + pathSnvFittingRData.toAbsolutePath() + "' with ID '" + hrdetectParams.getSnvFittingId() + "' for sample '" + + hrdetectParams.getSampleId() + "'"); + } + + pathSvFittingRData = getFittingRDataFile(svFitting.getFiles()); + if (!pathSvFittingRData.toFile().exists()) { + throw new ToolException("Unable to compute HRDetect analysis. No .rData file found for SV fitting '" + + pathSvFittingRData.toAbsolutePath() + "' with ID '" + hrdetectParams.getSvFittingId() + "' for sample '" + + hrdetectParams.getSampleId() + "'"); + } + + // Check CNV query + cnvQuery = JacksonUtils.getDefaultObjectMapper().readValue(hrdetectParams.getCnvQuery(), ObjectMap.class); + Individual individual = IndividualQcUtils.getIndividualBySampleId(getStudy(), hrdetectParams.getSampleId(), getCatalogManager(), + getToken()); + if (individual == null) { + throw new ToolException("Unable to compute HRDetect analysis. No individual found for sample '" + + hrdetectParams.getSampleId() + "', that individual must have at least two samples: somatic and germline"); + } + List samples = individual.getSamples(); + if (samples.size() < 2) { + throw new ToolException("For CNV query processing, individual (" + individual.getId() + ") must have at least two" + + " samples: somatic and germline"); + } + for (Sample sample : samples) { + if (!sample.isSomatic()) { + germlineSample = sample; + break; + } + } + if (germlineSample == null) { + throw new ToolException("Germline sample not found for individual '" + individual.getId() + "', it is mandatory for CNV query" + + " processing"); + } + + // Check INDEL query + indelQuery = JacksonUtils.getDefaultObjectMapper().readValue(hrdetectParams.getIndelQuery(), ObjectMap.class); + if (!indelQuery.containsKey(VariantQueryParam.SAMPLE.key())) { + logger.info("Setting sample in INDEL query"); + indelQuery.put(VariantQueryParam.SAMPLE.key(), somaticSample.getId()); + } + if (!somaticSample.getId().equals(indelQuery.getString(VariantQueryParam.SAMPLE.key()))) { + throw new ToolException("Mismatch sample from INDEL query '" + cnvQuery.getString(VariantQueryParam.SAMPLE.key())+ "' and" + + " sample '" + somaticSample.getId() + "'"); + } + + // Log messages + logger.info("HRDetect ID: {}", hrdetectParams.getId()); + logger.info("Study: {}", study); + logger.info("Assembly: {}", assembly); + logger.info("Somatatic sample ID: {}", somaticSample.getId()); + logger.info("Germline sample ID: {}", germlineSample.getId()); + logger.info("Signature fitting ID for SNV: {}", hrdetectParams.getSnvFittingId()); + logger.info("Signature fitting ID for SV: {}", hrdetectParams.getSvFittingId()); + logger.info("CNV query: {}", cnvQuery.toJson()); + logger.info("INDEL query: {}", indelQuery.toJson()); + logger.info("y (SNV3): {}", hrdetectParams.getSnv3CustomName()); + logger.info("z (SNV8): {}", hrdetectParams.getSnv8CustomName()); + logger.info("Y (SV3): {}", hrdetectParams.getSv3CustomName()); + logger.info("Z (SV8): {}", hrdetectParams.getSv8CustomName()); + logger.info("Bootstrap: {}", hrdetectParams.isBootstrap()); + } + + @Override + protected void run() throws ToolException { + step(getId(), () -> { + HRDetectAnalysisExecutor toolExecutor = getToolExecutor(HRDetectAnalysisExecutor.class); + + toolExecutor.setStudy(study) + .setSomaticSample(somaticSample.getId()) + .setGermlineSample(germlineSample.getId()) + .setAssembly(assembly) + .setSnvRDataPath(pathSnvFittingRData) + .setSvRDataPath(pathSvFittingRData) + .setCnvQuery(cnvQuery) + .setIndelQuery(indelQuery) + .setSnv3CustomName(hrdetectParams.getSnv3CustomName()) + .setSnv8CustomName(hrdetectParams.getSnv8CustomName()) + .setSv3CustomName(hrdetectParams.getSv3CustomName()) + .setSv8CustomName(hrdetectParams.getSv8CustomName()) + .setBootstrap(hrdetectParams.isBootstrap()) + .execute(); + + // Parse results and update quality control for the catalog sample + Sample sample = checkSample(hrdetectParams.getSampleId()); + HRDetect hrDetect = parseResult(getOutDir()); + SampleQualityControl qc = sample.getQualityControl(); + if (qc == null) { + qc = new SampleQualityControl(); + } + if (qc.getVariant() == null) { + qc.setVariant(new SampleVariantQualityControlMetrics()); + } + if (qc.getVariant().getHrDetects() == null) { + qc.getVariant().setHrDetects(new ArrayList<>()); + } + qc.getVariant().getHrDetects().add(hrDetect); + catalogManager.getSampleManager().update(getStudy(), sample.getId(), new SampleUpdateParams().setQualityControl(qc), + QueryOptions.empty(), getToken()); + }); + } + + public HRDetect parseResult(Path dir) throws IOException { + HRDetect result = new HRDetect() + .setId(hrdetectParams.getId()) + .setDescription(hrdetectParams.getDescription()) + .setSnvFittingId(hrdetectParams.getSnvFittingId()) + .setSvFittingId(hrdetectParams.getSvFittingId()) + .setCnvQuery(JacksonUtils.getDefaultObjectMapper().readValue(hrdetectParams.getCnvQuery(), ObjectMap.class)) + .setIndelQuery(JacksonUtils.getDefaultObjectMapper().readValue(hrdetectParams.getIndelQuery(), ObjectMap.class)); + + // Set other params + ObjectMap params = new ObjectMap(); + if (StringUtils.isNotEmpty(hrdetectParams.getSnv3CustomName())) { + params.append("snv3CustomName", hrdetectParams.getSnv3CustomName()); + } + if (StringUtils.isNotEmpty(hrdetectParams.getSnv8CustomName())) { + params.append("snv8CustomName", hrdetectParams.getSnv8CustomName()); + } + if (StringUtils.isNotEmpty(hrdetectParams.getSv3CustomName())) { + params.append("sv3CustomName", hrdetectParams.getSv3CustomName()); + } + if (StringUtils.isNotEmpty(hrdetectParams.getSv8CustomName())) { + params.append("sv8CustomName", hrdetectParams.getSv8CustomName()); + } + if (params.size() > 0) { + result.setParams(params); + } + + // Read scores + ObjectMap scores = new ObjectMap(); + File scoresFile = dir.resolve(HRDETECT_SCORES_FILENAME_DEFAULT).toFile(); + if (scoresFile.exists()) { + List lines = FileUtils.readLines(scoresFile, Charset.defaultCharset()); + if (lines.size() > 1) { + String[] labels = lines.get(0).split("\t"); + String[] values = lines.get(1).split("\t"); + for (int i = 0; i < labels.length; i++) { + try { + scores.put(labels[i], Float.parseFloat(values[i + 1])); + } catch (NumberFormatException e) { + scores.put(labels[i], Float.NaN); + } + } + } + } + if (MapUtils.isNotEmpty(scores)) { + result.setScores(scores); + } + + // TODO: files to be added ? + + return result; + } + + private Sample checkSample(String sampleId) throws ToolException, CatalogException { + study = catalogManager.getStudyManager().get(study, QueryOptions.empty(), token).first().getFqn(); + OpenCGAResult sampleResult = catalogManager.getSampleManager().get(study, sampleId, QueryOptions.empty(), token); + if (sampleResult.getNumResults() != 1) { + throw new ToolException("Unable to compute HRDetect analysis. Sample '" + hrdetectParams.getSampleId() + "' not found"); + } + + return sampleResult.first(); + } + + private void checkSampleQualityControl(Sample sample) throws ToolException { + if (sample.isSomatic()) { + // Check signatures are present in the quality control (only for somatic sample) + if (sample.getQualityControl() == null || sample.getQualityControl().getVariant() == null || + CollectionUtils.isEmpty(sample.getQualityControl().getVariant().getSignatures())) { + throw new ToolException("Unable to compute HRDetect analysis. No mutational signatures found for sample '" + + hrdetectParams.getSampleId() + "'"); + } + } + } + + private Path getFittingRDataFile(List files) { + if (CollectionUtils.isEmpty(files)) { + return null; + } + Path basePath = Paths.get(AnalysisUtils.getJobBaseDir(getOutDir().toAbsolutePath().toString())); + for (String file : files) { + if (file.endsWith("rData")) { + Path path = basePath.resolve(file); + logger.info("RData file found: {}; outdir = {}; path = {}", file, getOutDir().toAbsolutePath(), path.toAbsolutePath()); + return path; + } + } + return null; + } +} + diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java new file mode 100644 index 00000000000..4c96ad7b3b0 --- /dev/null +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/hrdetect/HRDetectLocalAnalysisExecutor.java @@ -0,0 +1,240 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.analysis.variant.hrdetect; + +import htsjdk.samtools.reference.BlockCompressedIndexedFastaSequenceFile; +import htsjdk.samtools.reference.FastaSequenceIndex; +import htsjdk.samtools.reference.ReferenceSequence; +import htsjdk.samtools.util.GZIIndex; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.biodata.models.variant.StudyEntry; +import org.opencb.biodata.models.variant.Variant; +import org.opencb.biodata.models.variant.avro.VariantType; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.datastore.core.QueryResultWriter; +import org.opencb.commons.exec.Command; +import org.opencb.commons.utils.DockerUtils; +import org.opencb.opencga.analysis.ResourceUtils; +import org.opencb.opencga.analysis.StorageToolExecutor; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.exceptions.ToolExecutorException; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.response.VariantQueryResult; +import org.opencb.opencga.core.tools.annotations.ToolExecutor; +import org.opencb.opencga.core.tools.variant.HRDetectAnalysisExecutor; +import org.opencb.opencga.core.tools.variant.MutationalSignatureAnalysisExecutor; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; +import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; +import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + +import static org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis.CATALOGUES_FILENAME_DEFAULT; + +@ToolExecutor(id="opencga-local", tool = HRDetectAnalysis.ID, + framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE) +public class HRDetectLocalAnalysisExecutor extends HRDetectAnalysisExecutor + implements StorageToolExecutor { + + public final static String R_DOCKER_IMAGE = "opencb/opencga-ext-tools:" + GitRepositoryState.get().getBuildVersion(); + + private final static String CNV_FILENAME = "cnv.tsv"; + private final static String INDEL_FILENAME = "indel.vcf"; + private final static String INDEL_SORTED_FILENAME = "indel.sorted.vcf"; + private final static String INDEL_GZ_FILENAME = "indel.sorted.vcf.gz"; + private final static String INPUT_TABLE_FILENAME = "inputTable.tsv"; + + private final static String VIRTUAL_VOLUMEN_DATA = "/data/"; + private final static String VIRTUAL_VOLUMEN_SNV = "/snv/"; + private final static String VIRTUAL_VOLUMEN_SV = "/sv/"; + + private Path opencgaHome; + + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Override + public void run() throws ToolException, CatalogException, IOException, StorageEngineException { + opencgaHome = Paths.get(getExecutorParams().getString("opencgaHome")); + + // Prepare CNV data + prepareCNVData(); + + // Prepare INDEL data + prepareINDELData(); + + // Prepare input table + prepareInputTable(); + + // Run R script for fitting signature + executeRScript(); + } + + private void prepareCNVData() throws ToolExecutorException, StorageEngineException, CatalogException, FileNotFoundException { + Query query = new Query(getCnvQuery()); + query.put(VariantQueryParam.STUDY.key(), getStudy()); + query.put(VariantQueryParam.SAMPLE.key(), getSomaticSample() + "," + getGermlineSample()); + + QueryOptions queryOptions = new QueryOptions(); + queryOptions.append(QueryOptions.INCLUDE, "id,studies"); + + logger.info("CNV query: {}", query); + logger.info("CNV query options: {}", queryOptions); + + PrintWriter pwOut = new PrintWriter(getOutDir().resolve("cnvs.discarded").toFile()); + + PrintWriter pw = new PrintWriter(getOutDir().resolve(CNV_FILENAME).toAbsolutePath().toString()); + pw.println("seg_no\tChromosome\tchromStart\tchromEnd\ttotal.copy.number.inNormal\tminor.copy.number.inNormal\t" + + "total.copy.number.inTumour\tminor.copy.number.inTumour"); + + VariantDBIterator iterator = getVariantStorageManager().iterator(query, queryOptions, getToken()); + int count = 0; + while (iterator.hasNext()) { + Variant variant = iterator.next(); + + if (CollectionUtils.isEmpty(variant.getStudies())) { + pwOut.println(variant.toStringSimple() + "\tStudies is empty"); + } else { + StudyEntry studyEntry = variant.getStudies().get(0); + try { + StringBuilder sb = new StringBuilder(++count) + .append("\t").append(variant.getChromosome()) + .append("\t").append(variant.getStart()) + .append("\t").append(variant.getEnd()) + .append("\t").append(Integer.parseInt(studyEntry.getSampleData(getGermlineSample(), "TCN"))) + .append("\t").append(Integer.parseInt(studyEntry.getSampleData(getGermlineSample(), "MCN"))) + .append("\t").append(Integer.parseInt(studyEntry.getSampleData(getSomaticSample(), "TCN"))) + .append("\t").append(Integer.parseInt(studyEntry.getSampleData(getSomaticSample(), "MCN"))); + + pw.println(sb); + } catch (NumberFormatException e) { + pwOut.println(variant.toStringSimple() + "\tError parsing TCN/MCN values: " + e.getMessage()); + } + } + } + + pw.close(); + pwOut.close(); + } + + private void prepareINDELData() throws ToolExecutorException, StorageEngineException, CatalogException, IOException { + Query query = new Query(getIndelQuery()); + query.put(VariantQueryParam.STUDY.key(), getStudy()); + + QueryOptions queryOptions = new QueryOptions(); + queryOptions.append(QueryOptions.INCLUDE, "id,studies") + .append(QueryOptions.SORT, true); + + logger.info("INDEL query: {}", query); + logger.info("INDEL query options: {}", queryOptions); + + getVariantStorageManager().exportData(getOutDir().resolve(INDEL_FILENAME).toAbsolutePath().toString(), + VariantWriterFactory.VariantOutputFormat.VCF, null, query, queryOptions, getToken()); + + if (!getOutDir().resolve(INDEL_FILENAME).toFile().exists()) { + new ToolExecutorException("Error exporting VCF file with INDEL variants"); + } + + // Workaround to sort, waiting for exporting to do it + File sortVcfFile = getOutDir().resolve("sort_vcf.sh").toFile(); + PrintWriter pw = new PrintWriter(sortVcfFile); + pw.println("#!/bin/sh"); + pw.println("cat $1 | awk '$1 ~ /^#/ {print $0;next} {print $0 | \"sort -k1,1 -k2,2n\"}' > $2"); + pw.close(); + new Command("bash " + sortVcfFile.getAbsolutePath() + + " " + getOutDir().resolve(INDEL_FILENAME).toAbsolutePath() + + " " + getOutDir().resolve(INDEL_SORTED_FILENAME).toFile()) + .run(); + sortVcfFile.delete(); + + // BGZIP + AbstractMap.SimpleEntry outputBinding = new AbstractMap.SimpleEntry<>(getOutDir() + .toAbsolutePath().toString(), VIRTUAL_VOLUMEN_DATA); + String cmdline = DockerUtils.run(R_DOCKER_IMAGE, null, outputBinding, "bgzip " + VIRTUAL_VOLUMEN_DATA + INDEL_SORTED_FILENAME, + null); + logger.info("Docker command line: " + cmdline); + + // TABIX + cmdline = DockerUtils.run(R_DOCKER_IMAGE, null, outputBinding, "tabix -p vcf " + VIRTUAL_VOLUMEN_DATA + INDEL_GZ_FILENAME, null); + logger.info("Docker command line: " + cmdline); + } + + private void prepareInputTable() throws FileNotFoundException { + PrintWriter pw = new PrintWriter(getOutDir().resolve(INPUT_TABLE_FILENAME).toAbsolutePath().toString()); + pw.println("sample\tIndels_vcf_files\tCNV_tab_files"); + pw.println(getSomaticSample() + "\t" + VIRTUAL_VOLUMEN_DATA + INDEL_GZ_FILENAME + "\t" + VIRTUAL_VOLUMEN_DATA + CNV_FILENAME); + pw.close(); + } + + private void executeRScript() throws IOException { + // Input + List> inputBindings = new ArrayList<>(); + inputBindings.add(new AbstractMap.SimpleEntry<>(getSnvRDataPath().toFile().getParent(), VIRTUAL_VOLUMEN_SNV)); + inputBindings.add(new AbstractMap.SimpleEntry<>(getSvRDataPath().toFile().getParent(), VIRTUAL_VOLUMEN_SV)); + + // Output + AbstractMap.SimpleEntry outputBinding = new AbstractMap.SimpleEntry<>(getOutDir() + .toAbsolutePath().toString(), VIRTUAL_VOLUMEN_DATA); + + // Command + StringBuilder scriptParams = new StringBuilder("R CMD Rscript --vanilla ") + .append("/opt/opencga/signature.tools.lib/scripts/hrDetect") + .append(" -x ").append(VIRTUAL_VOLUMEN_SNV).append(getSnvRDataPath().toFile().getName()) + .append(" -X ").append(VIRTUAL_VOLUMEN_SV).append(getSvRDataPath().toFile().getName()) + .append(" -i ").append(VIRTUAL_VOLUMEN_DATA).append(INPUT_TABLE_FILENAME) + .append(" -o ").append(VIRTUAL_VOLUMEN_DATA); + + if (StringUtils.isNotEmpty(getSnv3CustomName())) { + scriptParams.append(" -y ").append(getSnv3CustomName()); + } + if (StringUtils.isNotEmpty(getSnv8CustomName())) { + scriptParams.append(" -z ").append(getSnv8CustomName()); + } + if (StringUtils.isNotEmpty(getSv3CustomName())) { + scriptParams.append(" -Y ").append(getSv3CustomName()); + } + if (StringUtils.isNotEmpty(getSv8CustomName())) { + scriptParams.append(" -Z ").append(getSv3CustomName()); + } + if (getBootstrap() != null) { + scriptParams.append(" -b"); + } + + switch (getAssembly()) { + case "GRCh37": { + scriptParams.append(" --genomev=hg19"); + break; + } + case "GRCh38": { + scriptParams.append(" --genomev=hg38"); + break; + } + } + + String cmdline = DockerUtils.run(R_DOCKER_IMAGE, inputBindings, outputBinding, scriptParams.toString(), null); + logger.info("Docker command line: " + cmdline); + } +} diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java index 1a2f35154d9..6d01bb88cc2 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java @@ -32,8 +32,6 @@ import org.opencb.biodata.tools.variant.converters.ga4gh.Ga4ghVariantConverter; import org.opencb.biodata.tools.variant.converters.ga4gh.factories.AvroGa4GhVariantFactory; import org.opencb.biodata.tools.variant.converters.ga4gh.factories.ProtoGa4GhVariantFactory; -import org.opencb.cellbase.core.config.SpeciesProperties; -import org.opencb.cellbase.core.result.CellBaseDataResponse; import org.opencb.commons.datastore.core.*; import org.opencb.commons.datastore.core.result.Error; import org.opencb.commons.datastore.solr.SolrManager; @@ -551,11 +549,7 @@ public OpenCGAResult setCellbaseConfiguration(String project, CellBaseConfi engine.getConfiguration().setCellbase(cellbaseConfiguration); engine.reloadCellbaseConfiguration(); - CellBaseDataResponse species = engine.getCellBaseUtils().getCellBaseClient().getMetaClient().species(); - if (species == null || species.firstResult() == null) { - throw new IllegalArgumentException("Unable to access cellbase url '" + cellbaseConfiguration.getUrl() + "'" - + " version '" + cellbaseConfiguration.getVersion() + "'"); - } + engine.getCellBaseUtils().validateCellBaseConnection(); if (engine.getMetadataManager().exists()) { List jobDependsOn = new ArrayList<>(1); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantAnnotationOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantAnnotationOperationManager.java index 88cbff56756..204886426d4 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantAnnotationOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantAnnotationOperationManager.java @@ -124,7 +124,8 @@ private void synchronizeProjectMetadata(String projectStr, String token) throws Project project = catalogManager.getProjectManager().get(projectStr, null, token).first(); ProjectOrganism organism = project.getOrganism(); int currentRelease = project.getCurrentRelease(); - CatalogStorageMetadataSynchronizer.updateProjectMetadata(variantStorageEngine.getMetadataManager(), organism, currentRelease); + CatalogStorageMetadataSynchronizer.updateProjectMetadata(variantStorageEngine.getMetadataManager(), organism, currentRelease, + project.getCellbase()); } private String buildOutputFileName(String alias, String region) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java index bd4404f7118..046f8d4d13a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java @@ -167,7 +167,8 @@ private void updateProject(String studyFqn, String token) throws CatalogExceptio release = project.getCurrentRelease(); // Add species, assembly and release - CatalogStorageMetadataSynchronizer.updateProjectMetadata(variantStorageEngine.getMetadataManager(), project.getOrganism(), release); + CatalogStorageMetadataSynchronizer.updateProjectMetadata(variantStorageEngine.getMetadataManager(), project.getOrganism(), release, + project.getCellbase()); } /** diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java index 6653acb4730..4702fcfebae 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java @@ -33,6 +33,7 @@ import org.opencb.opencga.catalog.utils.FileMetadataReader; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.common.BatchUtils; +import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortStatus; import org.opencb.opencga.core.models.cohort.CohortUpdateParams; @@ -48,7 +49,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.*; -import org.opencb.opencga.storage.core.variant.annotation.annotators.AbstractCellBaseVariantAnnotator; +import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -113,12 +114,12 @@ public static void updateProjectMetadata(CatalogManager catalog, VariantStorageM sessionId) .first(); - updateProjectMetadata(scm, p.getOrganism(), p.getCurrentRelease()); + updateProjectMetadata(scm, p.getOrganism(), p.getCurrentRelease(), p.getCellbase()); } - public static void updateProjectMetadata(VariantStorageMetadataManager scm, ProjectOrganism organism, int release) + public static void updateProjectMetadata(VariantStorageMetadataManager scm, ProjectOrganism organism, int release, CellBaseConfiguration cellbase) throws StorageEngineException { - String scientificName = AbstractCellBaseVariantAnnotator.toCellBaseSpeciesName(organism.getScientificName()); + String scientificName = CellBaseUtils.toCellBaseSpeciesName(organism.getScientificName()); scm.updateProjectMetadata(projectMetadata -> { if (projectMetadata == null) { @@ -126,6 +127,7 @@ public static void updateProjectMetadata(VariantStorageMetadataManager scm, Proj } projectMetadata.setSpecies(scientificName); projectMetadata.setAssembly(organism.getAssembly()); + projectMetadata.setDataRelease(cellbase.getDataRelease()); projectMetadata.setRelease(release); return projectMetadata; }); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java index 6558baf3a86..c81b966dcaf 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureAnalysis.java @@ -16,22 +16,28 @@ package org.opencb.opencga.analysis.variant.mutationalSignature; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.biodata.models.clinical.qc.Signature; import org.opencb.biodata.models.clinical.qc.SignatureFitting; +import org.opencb.biodata.models.clinical.qc.SignatureFittingScore; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.AnalysisUtils; import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.exceptions.ToolExecutorException; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleQualityControl; import org.opencb.opencga.core.models.sample.SampleUpdateParams; +import org.opencb.opencga.core.models.sample.SampleVariantQualityControlMetrics; import org.opencb.opencga.core.models.variant.MutationalSignatureAnalysisParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.Tool; @@ -45,6 +51,9 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Locale; + +import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.STUDY; @Tool(id = MutationalSignatureAnalysis.ID, resource = Enums.Resource.VARIANT) public class MutationalSignatureAnalysis extends OpenCgaToolScopeStudy { @@ -53,22 +62,36 @@ public class MutationalSignatureAnalysis extends OpenCgaToolScopeStudy { public static final String DESCRIPTION = "Run mutational signature analysis for a given sample."; public final static String SIGNATURE_COEFFS_FILENAME = "exposures.tsv"; - public final static String SIGNATURE_FITTING_FILENAME = "signature_summary.png"; public final static String CATALOGUES_FILENAME_DEFAULT = "catalogues.tsv"; - - public final static String QC_UPDATE_KEYNAME = "qcUpdate"; + public final static String MUTATIONAL_SIGNATURE_DATA_MODEL_FILENAME = "mutational_signature.json"; + public final static String MUTATIONAL_SIGNATURE_FITTING_DATA_MODEL_FILENAME = "mutational_signature_fitting.json"; + + public static final String CLUSTERED = "clustered"; + public static final String NON_CLUSTERED = "non-clustered"; + public static final String LENGTH_NA = "na"; + public static final String LENGTH_1_10Kb= "1-10Kb"; + public static final String LENGTH_10Kb_100Kb = "10-100Kb"; + public static final String LENGTH_100Kb_1Mb = "100Kb-1Mb"; + public static final String LENGTH_1Mb_10Mb = "1Mb-10Mb"; + public static final String LENGTH_10Mb = ">10Mb"; + public static final String TYPE_DEL = "del"; + public static final String TYPE_TDS = "tds"; + public static final String TYPE_INV = "inv"; + public static final String TYPE_TRANS = "trans"; @ToolParams private MutationalSignatureAnalysisParams signatureParams = new MutationalSignatureAnalysisParams(); - private String sample; + private Sample sample; private String assembly; private ObjectMap query; - private String catalogues; private String signaturesFile; private String rareSignaturesFile; + private boolean runCatalogue = true; + private boolean runFitting = true; + @Override protected void check() throws Exception { super.check(); @@ -78,100 +101,108 @@ protected void check() throws Exception { throw new ToolException("Missing study"); } - // Two behaviours: using catalogues or using sample/query - if (StringUtils.isNotEmpty(signatureParams.getCatalogues())) { - // Fitting from file containing the counts - // Check if that file exists - OpenCGAResult fileResult = getCatalogManager().getFileManager().get(study, - signatureParams.getCatalogues(), QueryOptions.empty(), getToken()); - if (fileResult.getNumResults() == 0) { - throw new ToolException("Catalogues file '" + signatureParams.getCatalogues() + "' does not exist in study '" - + study + "'"); + if (StringUtils.isEmpty(signatureParams.getSample())) { + throw new ToolException("Missing sample. It is mandatory to run mutational signature analysis"); + } + + // Check sample + study = catalogManager.getStudyManager().get(study, QueryOptions.empty(), token).first().getFqn(); + OpenCGAResult sampleResult = catalogManager.getSampleManager().get(study, signatureParams.getSample(), + QueryOptions.empty(), token); + if (sampleResult.getNumResults() != 1) { + throw new ToolException("Unable to compute mutational signature analysis. Sample '" + signatureParams.getSample() + + "' not found"); + } + sample = sampleResult.first(); + + if (StringUtils.isNotEmpty(signatureParams.getSkip())) { + if (signatureParams.getSkip().contains(MutationalSignatureAnalysisParams.SIGNATURE_CATALOGUE_SKIP_VALUE)) { + runCatalogue = false; } - if (fileResult.getNumResults() > 1) { - throw new ToolException("Multiple files '" + signatureParams.getCatalogues() + "' found in study '" + study + "'"); + if (signatureParams.getSkip().contains(MutationalSignatureAnalysisParams.SIGNATURE_FITTING_SKIP_VALUE)) { + runFitting = false; } - catalogues = fileResult.first().getUri().toURL().getPath(); - logger.info("Signagture catalogues file: {}", catalogues); - } else if (StringUtils.isNotEmpty(signatureParams.getCataloguesContent())) { - // Fitting from counts - FileUtils.write(getOutDir().resolve(CATALOGUES_FILENAME_DEFAULT).toFile(), signatureParams.getCataloguesContent(), - Charset.defaultCharset(), false); - catalogues = getOutDir().resolve(CATALOGUES_FILENAME_DEFAULT).toString(); - logger.info("Signagture catalogues file: {}", catalogues); - } else { - // Fitting from sample/query + } + + // Check 'catalogue' processing + if (runCatalogue) { if (signatureParams.getQuery() == null) { - throw new ToolException("Missing signature query"); + throw new ToolException("Missing signature query. It is mandatory to compute mutational signature catalogue"); } + query = JacksonUtils.getDefaultObjectMapper().readValue(signatureParams.getQuery(), ObjectMap.class); - logger.info("Signagture query: {}", signatureParams.getQuery()); - if (!query.containsKey(VariantQueryParam.SAMPLE.key())) { - throw new ToolException("Missing sample in the signature query"); - } - if (StringUtils.isEmpty(query.getString(VariantQueryParam.SAMPLE.key()))) { - throw new ToolException("Sample is empty in the signature query"); + if (!query.containsKey(VariantQueryParam.SAMPLE.key()) + || StringUtils.isEmpty(query.getString(VariantQueryParam.SAMPLE.key()))) { + query.put(VariantQueryParam.SAMPLE.key(), signatureParams.getSample()); + } else { + // Check mismatch sample + String tmpSample = query.getString(VariantQueryParam.SAMPLE.key()); + if (tmpSample.contains(":")) { + tmpSample = tmpSample.split(":")[0]; + } + if (!tmpSample.equals(signatureParams.getSample())) { + throw new ToolException("Mismatch sample name, from sample parameter (" + signatureParams.getSample() + ", and from" + + " the query (" + query.getString(VariantQueryParam.SAMPLE.key()) + ")"); + } } + } - // Get sample - sample = query.getString(VariantQueryParam.SAMPLE.key()); - if (sample.contains(":")) { - sample = sample.split(":")[0]; + // Check 'fitting' processing + if (runFitting) { + if (StringUtils.isEmpty(signatureParams.getId())) { + throw new ToolException("Missing signature catalogue ID (counts ID). It is mandatory to compute signature fitting"); } - // Get assembly - assembly = ResourceUtils.getAssembly(catalogManager, study, token); - if (StringUtils.isEmpty(assembly)) { - throw new ToolException("Missing assembly for study '" + study + "'"); + // Check that signature (catalogue) ID exists for the sample + boolean found = false; + if (sample.getQualityControl() != null && sample.getQualityControl().getVariant() != null + && CollectionUtils.isNotEmpty(sample.getQualityControl().getVariant().getSignatures())) { + for (Signature signature : sample.getQualityControl().getVariant().getSignatures()) { + if (signatureParams.getId().equals(signature.getId())) { + found = true; + break; + } + } } - // TODO: improve this - switch (assembly.toUpperCase()) { - case "GRCH37": - assembly = "GRCh37"; - break; - case "GRCH38": - assembly = "GRCh38"; - break; - default: - break; + if (!found && !runCatalogue) { + throw new ToolException("Signature catalogue ID (counts ID) '" + signatureParams.getId() + "' not found for the sample" + + "'" + signatureParams.getSample() + "'"); } - try { - // Check sample - study = catalogManager.getStudyManager().get(study, QueryOptions.empty(), token).first().getFqn(); - OpenCGAResult sampleResult = catalogManager.getSampleManager().get(study, sample, QueryOptions.empty(), token); - if (sampleResult.getNumResults() != 1) { - throw new ToolException("Unable to compute mutational signature analysis. Sample '" + sample + "' not found"); - } - - // Check signatures file - if (StringUtils.isNotEmpty(signatureParams.getSignaturesFile())) { - org.opencb.opencga.core.models.file.File catalogFile = AnalysisUtils.getCatalogFile(signatureParams.getSignaturesFile(), - getStudy(), catalogManager.getFileManager(), getToken()); - signaturesFile = catalogFile.getUri().getPath(); - } + // Check signatures file + if (StringUtils.isNotEmpty(signatureParams.getFitSignaturesFile())) { + org.opencb.opencga.core.models.file.File catalogFile = AnalysisUtils.getCatalogFile(signatureParams.getFitSignaturesFile(), + getStudy(), catalogManager.getFileManager(), getToken()); + signaturesFile = catalogFile.getUri().getPath(); + } - // Check rare signatures file - if (StringUtils.isNotEmpty(signatureParams.getRareSignaturesFile())) { - org.opencb.opencga.core.models.file.File catalogFile = AnalysisUtils.getCatalogFile( - signatureParams.getRareSignaturesFile(), getStudy(), catalogManager.getFileManager(), getToken()); - rareSignaturesFile = catalogFile.getUri().getPath(); - } - } catch (CatalogException e) { - throw new ToolException(e); + // Check rare signatures file + if (StringUtils.isNotEmpty(signatureParams.getFitRareSignaturesFile())) { + org.opencb.opencga.core.models.file.File catalogFile = AnalysisUtils.getCatalogFile( + signatureParams.getFitRareSignaturesFile(), getStudy(), catalogManager.getFileManager(), getToken()); + rareSignaturesFile = catalogFile.getUri().getPath(); } } + // Get assembly + assembly = getAssembly(study, catalogManager, token); + // Log messages - logger.info("Signagture fitting method: {}", signatureParams.getFitMethod()); - logger.info("Signagture sig. version: {}", signatureParams.getSigVersion()); - logger.info("Signagture organ: {}", signatureParams.getOrgan()); - logger.info("Signagture n boot: {}", signatureParams.getnBoot()); - logger.info("Signagture threshold percentage: {}", signatureParams.getThresholdPerc()); - logger.info("Signagture threshold p-value: {}", signatureParams.getThresholdPval()); - logger.info("Signagture max. rare sigs.: {}", signatureParams.getMaxRareSigs()); - logger.info("Signagture signatures file: {}", signaturesFile); - logger.info("Signagture rare signatures file: {}", rareSignaturesFile); + logger.info("Signagture id: {}", signatureParams.getId()); + logger.info("Signagture description: {}", signatureParams.getDescription()); + logger.info("Signagture sample: {}", signatureParams.getSample()); + logger.info("Signagture query: {}", signatureParams.getQuery()); + logger.info("Signagture fit id: {}", signatureParams.getFitId()); + logger.info("Signagture fit method: {}", signatureParams.getFitMethod()); + logger.info("Signagture fit sig. version: {}", signatureParams.getFitSigVersion()); + logger.info("Signagture fit organ: {}", signatureParams.getFitOrgan()); + logger.info("Signagture fit n boot: {}", signatureParams.getFitNBoot()); + logger.info("Signagture fit threshold percentage: {}", signatureParams.getFitThresholdPerc()); + logger.info("Signagture fit threshold p-value: {}", signatureParams.getFitThresholdPval()); + logger.info("Signagture fit max. rare sigs.: {}", signatureParams.getFitMaxRareSigs()); + logger.info("Signagture fit signatures file: {}", signaturesFile); + logger.info("Signagture fit rare signatures file: {}", rareSignaturesFile); + logger.info("Skip: {}", signatureParams.getSkip()); } @Override @@ -180,135 +211,250 @@ protected void run() throws ToolException { MutationalSignatureAnalysisExecutor toolExecutor = getToolExecutor(MutationalSignatureAnalysisExecutor.class); toolExecutor.setStudy(study) - .setSample(sample) + .setSample(signatureParams.getSample()) .setAssembly(assembly) .setQueryId(signatureParams.getId()) .setQueryDescription(signatureParams.getDescription()) + .setSample(signatureParams.getSample()) .setQuery(query) - .setCatalogues(catalogues) + .setFitId(signatureParams.getFitId()) .setFitMethod(signatureParams.getFitMethod()) - .setSigVersion(signatureParams.getSigVersion()) - .setOrgan(signatureParams.getOrgan()) - .setnBoot(signatureParams.getnBoot()) - .setThresholdPerc(signatureParams.getThresholdPerc()) - .setThresholdPval(signatureParams.getThresholdPval()) - .setMaxRareSigs(signatureParams.getMaxRareSigs()) + .setSigVersion(signatureParams.getFitSigVersion()) + .setOrgan(signatureParams.getFitOrgan()) + .setnBoot(signatureParams.getFitNBoot()) + .setThresholdPerc(signatureParams.getFitThresholdPerc()) + .setThresholdPval(signatureParams.getFitThresholdPval()) + .setMaxRareSigs(signatureParams.getFitMaxRareSigs()) .setSignaturesFile(signaturesFile) .setRareSignaturesFile(rareSignaturesFile) + .setSkip(signatureParams.getSkip()) .execute(); + }); + + // Get sample quality control again in case it was updated during the mutational signature analysis + OpenCGAResult sampleResult; + try { + sampleResult = catalogManager.getSampleManager().get(study, signatureParams.getSample(), + QueryOptions.empty(), token); + } catch (CatalogException e) { + throw new ToolException("After mutational signature analysis, it could not get sample from OpenCGA catalog", e); + } + if (sampleResult.getNumResults() != 1) { + throw new ToolException("After mutational signature analysis, it could not get sample '" + signatureParams.getSample() + "'" + + " from OpenCGA catalog: number of occurrences found: " + sampleResult.getNumResults()); + } - // Update quality control for the catalog sample - if (signatureParams.getQuery() != null && query.containsKey(QC_UPDATE_KEYNAME)) { - // Remove quality control update key - query.remove(QC_UPDATE_KEYNAME); + if (StringUtils.isEmpty(signatureParams.getId())) { + // Nothing to do + return; + } - OpenCGAResult sampleResult = getCatalogManager().getSampleManager().get(getStudy(), sample, QueryOptions.empty(), - getToken()); - Sample sample = sampleResult.first(); - if (sample != null) { + // Only save results in sample quality control, if the signature ID is not empty + sample = sampleResult.first(); + SampleQualityControl qc = sample.getQualityControl(); - Signature signature = parse(getOutDir()); - SampleQualityControl qc = sampleResult.first().getQualityControl(); - if (qc == null) { - qc = new SampleQualityControl(); + // Sanity check + if (qc == null) { + qc = new SampleQualityControl(); + } + if (qc.getVariant() == null) { + qc.setVariant(new SampleVariantQualityControlMetrics()); + } + if (qc.getVariant().getSignatures() == null) { + qc.getVariant().setSignatures(new ArrayList<>()); + } + + Signature signature = null; + SignatureFitting signatureFitting = null; + try { + File signatureFile = getOutDir().resolve(MUTATIONAL_SIGNATURE_DATA_MODEL_FILENAME).toFile(); + if (signatureFile.exists()) { + signature = JacksonUtils.getDefaultObjectMapper().readerFor(Signature.class).readValue(signatureFile); + } + File signatureFittingFile = getOutDir().resolve(MUTATIONAL_SIGNATURE_FITTING_DATA_MODEL_FILENAME).toFile(); + if (signatureFittingFile.exists()) { + signatureFitting = JacksonUtils.getDefaultObjectMapper().readerFor(SignatureFitting.class).readValue(signatureFittingFile); + } + } catch (IOException e) { + throw new ToolException("Something happened when parsing result files from mutational signature (or fitting)", e); + } + if (signature != null) { + logger.info("Adding new mutational signature to the signature data model before saving quality control"); + qc.getVariant().getSignatures().add(signature); + } + if (signatureFitting != null) { + for (Signature sig : qc.getVariant().getSignatures()) { + if (StringUtils.isNotEmpty(sig.getId())) { + if (sig.getId().equals(signatureParams.getId())) { + if (CollectionUtils.isEmpty(sig.getFittings())) { + sig.setFittings(new ArrayList<>()); + } + logger.info("Fitting {} was added to the mutational siganture {} before saving quality control", + signatureParams.getFitId(), signatureParams.getId()); + sig.getFittings().add(signatureFitting); + break; } - qc.getVariant().getSignatures().add(signature); + } + } + } + // Update sample quality control + try { + catalogManager.getSampleManager().update(getStudy(), sample.getId(), new SampleUpdateParams().setQualityControl(qc), + QueryOptions.empty(), getToken()); + logger.info("Quality control saved for sample {}", sample.getId()); + } catch (CatalogException e) { + throw new ToolException("Something happened when saving sample quality control", e); + } + } + + public static String getContextIndexFilename(String sample, String assembly) { + return "OPENCGA_" + sample + "_" + assembly + "_genome_context.csv"; + } - catalogManager.getSampleManager().update(getStudy(), sample.getId(), new SampleUpdateParams().setQualityControl(qc), - QueryOptions.empty(), getToken()); + public static String getAssembly(String study, CatalogManager catalogManager, String token) throws CatalogException, ToolException { + String assembly = ResourceUtils.getAssembly(catalogManager, study, token); + if (StringUtils.isEmpty(assembly)) { + throw new ToolException("Missing assembly for study '" + study + "'"); + } + // TODO: improve this + switch (assembly.toUpperCase()) { + case "GRCH37": + assembly = "GRCh37"; + break; + case "GRCH38": + assembly = "GRCh38"; + break; + default: + break; + } + return assembly; + } + + public static File getGenomeContextFile(String sample, String study, CatalogManager catalogManager, String token) + throws CatalogException, ToolException { + File indexFile = null; + String assembly = MutationalSignatureAnalysis.getAssembly(study, catalogManager, token); + String indexFilename = getContextIndexFilename(sample, assembly); + try { + Query fileQuery = new Query("name", indexFilename); + QueryOptions fileQueryOptions = new QueryOptions("include", "uri"); + OpenCGAResult fileResult = catalogManager.getFileManager().search(study, fileQuery, + fileQueryOptions, token); + + long maxSize = 0; + for (org.opencb.opencga.core.models.file.File file : fileResult.getResults()) { + File auxFile = new File(file.getUri().getPath()); + if (auxFile.exists() && auxFile.length() > maxSize) { + maxSize = auxFile.length(); + indexFile = auxFile; } } - }); + } catch (CatalogException e) { + throw new ToolExecutorException(e); + } + + return indexFile; } - public Signature parse(Path dir) throws IOException { - Signature result = new Signature(signatureParams.getId(), signatureParams.getDescription(), query, "SNV", null, null, null); + public static List parseCatalogueResults(Path dir) throws IOException { + List sigCounts = null; // Context counts File contextFile = dir.resolve(CATALOGUES_FILENAME_DEFAULT).toFile(); if (contextFile.exists()) { List lines = FileUtils.readLines(contextFile, Charset.defaultCharset()); - List sigCounts = new ArrayList<>(lines.size() - 1); + sigCounts = new ArrayList<>(lines.size() - 1); for (int i = 1; i < lines.size(); i++) { String[] fields = lines.get(i).split("\t"); sigCounts.add(new Signature.GenomeContextCount(fields[0], Math.round(Float.parseFloat((fields[1]))))); } - result.setCounts(sigCounts); + } + + return sigCounts; + } + + public static SignatureFitting parseFittingResults(Path outDir, String fitId, String fitMethod, String fitSigVersion, Integer fitNBoot, + String fitOrgan, Float fitThresholdPerc, Float fitThresholdPval, + Integer fitMaxRareSigs) throws IOException { + // Check for fitting coeffs. file + File coeffsFile = outDir.resolve(SIGNATURE_COEFFS_FILENAME).toFile(); + if (!coeffsFile.exists()) { + return null; } // Signature fitting - File coeffsFile = dir.resolve(SIGNATURE_COEFFS_FILENAME).toFile(); - if (coeffsFile.exists()) { - SignatureFitting fitting = new SignatureFitting() - .setMethod(signatureParams.getFitMethod()) - .setSignatureVersion(signatureParams.getSigVersion()); - - // Set source from fit method - if (StringUtils.isNotEmpty(getSignatureParams().getSigVersion())) { - if (getSignatureParams().getSigVersion().startsWith("COSMIC")) { - fitting.setSignatureSource("COSMIC"); - } else if (getSignatureParams().getSigVersion().startsWith("RefSig")) { - fitting.setSignatureSource("RefSig"); - } + SignatureFitting fitting = new SignatureFitting(); + if (StringUtils.isNotEmpty(fitId)) { + fitting.setId(fitId); + } + if (StringUtils.isNotEmpty(fitMethod)) { + fitting.setMethod(fitMethod); + } + if (StringUtils.isNotEmpty(fitSigVersion)) { + fitting.setSignatureVersion(fitSigVersion); + if (fitSigVersion.startsWith("COSMIC")) { + fitting.setSignatureSource("COSMIC"); + } else if (fitSigVersion.startsWith("RefSig")) { + fitting.setSignatureSource("RefSig"); } + } - // Set fitting scores - List lines = FileUtils.readLines(coeffsFile, Charset.defaultCharset()); - String[] labels = lines.get(0).split("\t"); - String[] values = lines.get(1).split("\t"); - List scores = new ArrayList<>(labels.length); - for (int i = 0; i < labels.length; i++) { - String label = labels[i]; - if (label.contains("_")) { - String[] splits = label.split("_"); - label = splits[splits.length - 1]; - } - scores.add(new SignatureFitting.Score(label, Double.parseDouble(values[i + 1]))); + // Set fitting scores + List lines = FileUtils.readLines(coeffsFile, Charset.defaultCharset()); + String[] labels = lines.get(0).split("\t"); + String[] values = lines.get(1).split("\t"); + List scores = new ArrayList<>(labels.length); + for (int i = 0; i < labels.length; i++) { + String label = labels[i]; + if (label.contains("_")) { + String[] splits = label.split("_"); + label = splits[splits.length - 1]; } - fitting.setScores(scores); - - // Set files - List files = new ArrayList<>(); - for (File file : getOutDir().toFile().listFiles()) { - if (file.getName().endsWith("pdf")) { - files.add(file.getName()); - } else if (file.isDirectory()) { - for (File file2 : file.listFiles()) { - if (file2.getName().endsWith("pdf")) { - files.add(file.getName() + "/" + file2.getName()); - } + scores.add(new SignatureFittingScore(label, Double.parseDouble(values[i + 1]))); + } + fitting.setScores(scores); + + // Set files + List files = new ArrayList<>(); + for (File file : outDir.toFile().listFiles()) { + if (file.getName().equals("catalogues.pdf")) { + continue; + } + if (file.getName().endsWith("pdf") || file.getName().equals("fitData.rData")) { + files.add(AnalysisUtils.getJobFileRelativePath(file.getAbsolutePath())); + } else if (file.isDirectory()) { + for (File file2 : file.listFiles()) { + if (file2.getName().endsWith("pdf")) { + files.add(AnalysisUtils.getJobFileRelativePath(file2.getAbsolutePath())); } } } - fitting.setFiles(files); + } + fitting.setFiles(files); - // Set params - ObjectMap params = new ObjectMap(); - if (signatureParams.getnBoot() != null) { - params.append("nBoot", signatureParams.getnBoot()); - } - if (StringUtils.isNotEmpty(signatureParams.getOrgan())) { - params.append("organ", signatureParams.getOrgan()); - } - if (signatureParams.getThresholdPerc() != null) { - params.append("thresholdPerc", signatureParams.getThresholdPerc()); - } - if (signatureParams.getThresholdPval() != null) { - params.append("thresholdPval", signatureParams.getThresholdPval()); - } - if (signatureParams.getMaxRareSigs() != null) { - params.append("maxRareSigs", signatureParams.getMaxRareSigs()); - } - if (params.size() > 0) { - fitting.setParams(params); - } + // Set params + ObjectMap params = new ObjectMap(); + if (fitNBoot != null) { + params.append("nBoot", fitNBoot); + } + if (StringUtils.isNotEmpty(fitOrgan)) { + params.append("organ", fitOrgan); + } + if (fitThresholdPerc != null) { + params.append("thresholdPerc", fitThresholdPerc); + } + if (fitThresholdPval != null) { + params.append("thresholdPval", fitThresholdPval); + } + if (fitMaxRareSigs != null) { + params.append("maxRareSigs", fitMaxRareSigs); + } + if (params.size() > 0) { fitting.setParams(params); - - // Set fitting signature - result.setFitting(fitting); } + fitting.setParams(params); - return result; + return fitting; } public MutationalSignatureAnalysisParams getSignatureParams() { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java index f0816d528e5..55173c3bd3a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java @@ -20,18 +20,28 @@ import htsjdk.samtools.reference.FastaSequenceIndex; import htsjdk.samtools.reference.ReferenceSequence; import htsjdk.samtools.util.GZIIndex; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.opencb.biodata.models.clinical.qc.Signature; +import org.opencb.biodata.models.clinical.qc.SignatureFitting; import org.opencb.biodata.models.variant.Variant; +import org.opencb.biodata.models.variant.avro.BreakendMate; +import org.opencb.biodata.models.variant.avro.FileEntry; import org.opencb.biodata.models.variant.avro.VariantType; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.DockerUtils; +import org.opencb.commons.utils.FileUtils; import org.opencb.opencga.analysis.ResourceUtils; import org.opencb.opencga.analysis.StorageToolExecutor; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.ToolExecutorException; +import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.variant.MutationalSignatureAnalysisParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.core.tools.annotations.ToolExecutor; @@ -47,7 +57,7 @@ import java.nio.file.Paths; import java.util.*; -import static org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis.CATALOGUES_FILENAME_DEFAULT; +import static org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis.*; @ToolExecutor(id="opencga-local", tool = MutationalSignatureAnalysis.ID, framework = ToolExecutor.Framework.LOCAL, source = ToolExecutor.Source.STORAGE) @@ -65,114 +75,83 @@ public class MutationalSignatureLocalAnalysisExecutor extends MutationalSignatur public void run() throws ToolException, CatalogException, IOException, StorageEngineException { opencgaHome = Paths.get(getExecutorParams().getString("opencgaHome")); - if (StringUtils.isEmpty(getCatalogues())) { + // Check genome context file for that sample, and create it if necessary + if (StringUtils.isNotEmpty(getSkip()) + && getSkip().contains(MutationalSignatureAnalysisParams.SIGNATURE_CATALOGUE_SKIP_VALUE) + && getSkip().contains(MutationalSignatureAnalysisParams.SIGNATURE_FITTING_SKIP_VALUE)) { + // Only compute genome context file + // TODO: overwrite support ! + File indexFile = checkGenomeContextFile(); + logger.info("Checking genome context file {} for sample {}", indexFile.getAbsolutePath(), getSample()); + } + + if (StringUtils.isEmpty(getSkip()) || (!getSkip().contains(MutationalSignatureAnalysisParams.SIGNATURE_CATALOGUE_SKIP_VALUE))) { // Get first variant to check where the genome context is stored Query query = new Query(); if (getQuery() != null) { query.putAll(getQuery()); } - // Ovewrite study and type (SNV) - query.append(VariantQueryParam.STUDY.key(), getStudy()).append(VariantQueryParam.TYPE.key(), VariantType.SNV); - - QueryOptions queryOptions = new QueryOptions(); - queryOptions.append(QueryOptions.INCLUDE, "id"); - queryOptions.append(QueryOptions.LIMIT, "1"); + // Overwrite study and type (SNV) + String type = query.getString(VariantQueryParam.TYPE.key()); + if (type.equals(VariantType.SNV.name())) { + // SNV + logger.info("Computing catalogue (mutational signature) for SNV variants"); + + // TODO: overwrite support ! + File indexFile = checkGenomeContextFile(); + logger.info("Mutational signature analysis is using the genome context file {} for sample {}", indexFile.getAbsolutePath(), + getSample()); + + query.append(VariantQueryParam.STUDY.key(), getStudy()).append(VariantQueryParam.TYPE.key(), VariantType.SNV); + + QueryOptions queryOptions = new QueryOptions(); + queryOptions.append(QueryOptions.INCLUDE, "id"); + queryOptions.append(QueryOptions.LIMIT, "1"); + + VariantQueryResult variantQueryResult = getVariantStorageManager().get(query, queryOptions, getToken()); + Variant variant = variantQueryResult.first(); + if (variant == null) { + // Nothing to do + addWarning("None variant found for that mutational signature query"); + return; + } - VariantQueryResult variantQueryResult = getVariantStorageManager().get(query, queryOptions, getToken()); - Variant variant = variantQueryResult.first(); - if (variant == null) { - // Nothing to do - addWarning("None variant found for that mutational signature query"); - return; + // Run mutational analysis taking into account that the genome context is stored in an index file, + // if the genome context file does not exist, it will be created !!! + computeSignatureCatalogueSNV(indexFile); + } else { + // SV + logger.info("Computing catalogue (mutational signature) for SV variants"); + computeSignatureCatalogueSV(); } - - // Run mutational analysis taking into account that the genome context is stored in an index file, - // if the genome context file does not exist, it will be created !!! - computeFromContextFile(); } - // Run R script for fitting signature - executeRScript(); -// if (StringUtils.isEmpty(getOrgan()) && (StringUtils.isEmpty(getSigVersion()) || getSigVersion().startsWith("Ref"))) { -// addWarning("Since the parameter 'organ' is missing and RefSig is been used, the fitting signature will not be computed."); -// } else { -// executeRScript(); -// } + if (StringUtils.isEmpty(getSkip()) || (!getSkip().contains(MutationalSignatureAnalysisParams.SIGNATURE_FITTING_SKIP_VALUE))) { + // Run R script for fitting signature + computeSignatureFitting(); + } } - private void computeFromContextFile() throws ToolExecutorException { + private File checkGenomeContextFile() throws ToolExecutorException { // Context index filename - File indexFile = null; - String indexFilename = getContextIndexFilename(); + File indexFile; try { - Query fileQuery = new Query("name", indexFilename); - QueryOptions fileQueryOptions = new QueryOptions("include", "uri"); - OpenCGAResult fileResult = getVariantStorageManager() - .getCatalogManager() - .getFileManager().search(getStudy(), fileQuery, fileQueryOptions, getToken()); - - long maxSize = 0; - for (org.opencb.opencga.core.models.file.File file : fileResult.getResults()) { - File auxFile = new File(file.getUri().getPath()); - if (auxFile.exists() && auxFile.length() > maxSize) { - maxSize = auxFile.length(); - indexFile = auxFile; - } - } - } catch (CatalogException e) { - throw new ToolExecutorException(e); + indexFile = MutationalSignatureAnalysis.getGenomeContextFile(getSample(), getStudy(), getVariantStorageManager().getCatalogManager(), getToken()); + } catch (CatalogException | ToolException e) { + indexFile = null; } - - if (indexFile == null) { - // The genome context file does not exist, we have to create it !!! - indexFile = getOutDir().resolve(indexFilename).toFile(); - createGenomeContextFile(indexFile); + if (indexFile != null && indexFile.exists()) { + return indexFile; } + // The genome context file does not exist, we have to create it !!! + indexFile = getOutDir().resolve(MutationalSignatureAnalysis.getContextIndexFilename(getSample(), getAssembly())).toFile(); + createGenomeContextFile(indexFile); + if (!indexFile.exists()) { throw new ToolExecutorException("Could not create the genome context index file for sample " + getSample()); } - - try { - // Read context index - Map indexMap = new HashMap<>(); - BufferedReader br = new BufferedReader(new FileReader(indexFile)); - String line; - while ((line = br.readLine()) != null) { - String[] parts = line.split("\t"); - indexMap.put(parts[0], parts[1]); - } - - // Get variant iterator - Query query = new Query(); - if (getQuery() != null) { - query.putAll(getQuery()); - } - // Ovewrite study and type (SNV) - query.append(VariantQueryParam.STUDY.key(), getStudy()).append(VariantQueryParam.TYPE.key(), VariantType.SNV); - - QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, "id"); - - VariantDBIterator iterator = getVariantStorageManager().iterator(query, queryOptions, getToken()); - - Map> countMap = initCountMap(); - - while (iterator.hasNext()) { - Variant variant = iterator.next(); - - // Update count map - updateCountMap(variant, indexMap.get(variant.toString()), countMap); - } - - // Write context counts - File cataloguesFile = getOutDir().resolve(CATALOGUES_FILENAME_DEFAULT).toFile(); - writeCountMap(getSample(), countMap, cataloguesFile); - - // Update the parameter catalogues - setCatalogues(cataloguesFile.getAbsolutePath()); - } catch (IOException | CatalogException | StorageEngineException | ToolException e) { - throw new ToolExecutorException(e); - } + return indexFile; } private void createGenomeContextFile(File indexFile) throws ToolExecutorException { @@ -252,13 +231,312 @@ private void updateCountMap(Variant variant, String sequence, Map indexMap = new HashMap<>(); + BufferedReader br = new BufferedReader(new FileReader(indexFile)); + String line; + while ((line = br.readLine()) != null) { + String[] parts = line.split("\t"); + indexMap.put(parts[0], parts[1]); + } + + // Get variant iterator + Query query = new Query(); + if (getQuery() != null) { + query.putAll(getQuery()); + } + // Ovewrite study and type (SNV) + query.append(VariantQueryParam.STUDY.key(), getStudy()).append(VariantQueryParam.TYPE.key(), VariantType.SNV); + + QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, "id"); + + VariantDBIterator iterator = getVariantStorageManager().iterator(query, queryOptions, getToken()); + + Map> countMap = initCountMap(); + + while (iterator.hasNext()) { + Variant variant = iterator.next(); + + // Update count map + updateCountMap(variant, indexMap.get(variant.toString()), countMap); + } + + // Write context counts + File cataloguesFile = getOutDir().resolve(CATALOGUES_FILENAME_DEFAULT).toFile(); + writeCountMap(getSample(), countMap, cataloguesFile); + + // Check catalogue file before parsing and creating the mutational signature data model + if (!cataloguesFile.exists()) { + throw new ToolExecutorException("Something wrong happened: counts file " + CATALOGUES_FILENAME_DEFAULT + " could not be" + + " generated"); + } + List genomeContextCounts = parseCatalogueResults(getOutDir()); + Signature signature = new Signature() + .setId(getQueryId()) + .setDescription(getQueryDescription()) + .setQuery(query) + .setType("SNV") + .setCounts(genomeContextCounts); + + JacksonUtils.getDefaultObjectMapper().writerFor(Signature.class).writeValue(getOutDir() + .resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_DATA_MODEL_FILENAME).toFile(), signature); + } catch (IOException | CatalogException | StorageEngineException | ToolException e) { + throw new ToolExecutorException(e); + } + } + + public void computeSignatureCatalogueSV() throws ToolExecutorException { + try { + // Get variant iterator + Query query = new Query(); + if (getQuery() != null) { + query.putAll(getQuery()); + } + // Overwrite study and types related to SV + query.put(VariantQueryParam.STUDY.key(), getStudy()); + query.put(VariantQueryParam.TYPE.key(), VariantType.DELETION + "," + VariantType.BREAKEND + "," + VariantType.DUPLICATION + "," + + VariantType.TANDEM_DUPLICATION + "," + VariantType.INVERSION + "," + VariantType.TRANSLOCATION); + + QueryOptions queryOptions = new QueryOptions(QueryOptions.INCLUDE, "id,sv,studies"); + + logger.info("Query: {}", query.toJson()); + logger.info("Query options: {}", queryOptions.toJson()); + + File clusteredFile = computeClusteredFile(query, queryOptions); + + BufferedReader br = FileUtils.newBufferedReader(clusteredFile.toPath()); + // Skip header line + // chrom1 start1 end1 chrom2 start2 end2 length type sample id is.clustered + // 0 1 2 3 4 5 6 7 8 9 10 + Map countMap = new HashMap<>(); + // Skip first line + String line = br.readLine(); + while ((line = br.readLine()) != null) { + String[] split = line.split("\t"); + if (split.length != 11) { + logger.warn("Skipping line {}: it does not contain 11 elements", line); + continue; + } + String clusteredKey = split[10].equals("FALSE") ? NON_CLUSTERED : CLUSTERED; + String lengthKey = split[6]; + String typeKey = split[7]; + + String key = clusteredKey + "_" + typeKey; + if (!typeKey.equals(TYPE_TRANS)) { + key += ("_" + lengthKey); + } + if (countMap.containsKey(key)) { + countMap.put(key, 1 + countMap.get(key)); + } else { + countMap.put(key, 1); + } + } + +// logger.info("Count map size = {}", countMap.size()); +// for (Map.Entry entry : countMap.entrySet()) { +// logger.info("context = {}, count = {}", entry.getKey(), entry.getValue()); +// } + + // Build teh genome context counts object for SV + List genomeContextCounts = new LinkedList<>(); + for (String clustered: new LinkedList<>(Arrays.asList(CLUSTERED, NON_CLUSTERED))) { + for (String type: new LinkedList<>(Arrays.asList(TYPE_DEL, TYPE_TDS, TYPE_INV))) { + for (String length : new LinkedList<>(Arrays.asList(LENGTH_1_10Kb, LENGTH_10Kb_100Kb, LENGTH_100Kb_1Mb, LENGTH_1Mb_10Mb, + LENGTH_10Mb))) { + String key = clustered + "_" + type + "_" + length; + genomeContextCounts.add(new Signature.GenomeContextCount(key, countMap.containsKey(key) ? countMap.get(key) : 0)); + } + } + String key = clustered + "_" + TYPE_TRANS; + genomeContextCounts.add(new Signature.GenomeContextCount(key, countMap.containsKey(key) ? countMap.get(key) : 0)); + } + + // Write catalogue file from the genome context counts + PrintWriter pw = new PrintWriter(getOutDir().resolve(CATALOGUES_FILENAME_DEFAULT).toFile()); + pw.write(query.getString(VariantQueryParam.SAMPLE.key())); + pw.write("\n"); + for (Signature.GenomeContextCount counts : genomeContextCounts) { + pw.write(counts.getContext() + "\t" + counts.getTotal() + "\n"); + } + pw.close(); + + Signature signature = new Signature() + .setId(getQueryId()) + .setDescription(getQueryDescription()) + .setQuery(query) + .setType("SV") + .setCounts(genomeContextCounts); + + JacksonUtils.getDefaultObjectMapper().writerFor(Signature.class).writeValue(getOutDir() + .resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_DATA_MODEL_FILENAME).toFile(), signature); + } catch (IOException | CatalogException | StorageEngineException | ToolException e) { + throw new ToolExecutorException(e); + } + } + + private File computeClusteredFile(Query query, QueryOptions queryOptions) throws ToolException, StorageEngineException, + CatalogException { + VariantDBIterator iterator = getVariantStorageManager().iterator(query, queryOptions, getToken()); + + // $ Rscript sv_clustering.R ./test.bedpe ./out.bedpe + File inputFile = getOutDir().resolve("in.clustered.bedpe").toFile(); + File outputFile = getOutDir().resolve("out.clustered.bedpe").toFile(); + try { + PrintWriter pw = new PrintWriter(inputFile); + pw.println("chrom1\tstart1\tend1\tchrom2\tstart2\tend2\tlength\ttype\tsample"); + while (iterator.hasNext()) { + Variant variant = iterator.next(); + if (variant.getSv() == null || variant.getSv().getBreakend() == null || variant.getSv().getBreakend().getMate() == null) { + continue; + } + String typeKey = getTypeKey(variant); + String lengthKey = getLengthKey(variant); + if (typeKey != null && lengthKey != null) { + BreakendMate mate = variant.getSv().getBreakend().getMate(); + pw.println(variant.getChromosome() + "\t" + variant.getStart() + "\t" + variant.getEnd() + "\t" + + mate.getChromosome() + "\t" + mate.getPosition() + "\t" + mate.getPosition() + "\t" + + lengthKey + "\t" + typeKey + "\t" + getSample()); + } + } + pw.close(); + + // Build command line to run R script via docker image + // Input binding + List> inputBindings = new ArrayList<>(); + inputBindings.add(new AbstractMap.SimpleEntry<>(opencgaHome.resolve("analysis/" + MutationalSignatureAnalysis.ID) + .toAbsolutePath().toString(), "/script")); + + // Output binding + AbstractMap.SimpleEntry outputBinding = new AbstractMap.SimpleEntry<>(getOutDir().toAbsolutePath().toString(), + "/jobdir"); + + String rParams = "R CMD Rscript --vanilla /script/sv_clustering.R" + + " /jobdir/" + inputFile.getName() + + " /jobdir/" + outputFile.getName(); + + // Execute R script in docker + DockerUtils.run(MutationalSignatureLocalAnalysisExecutor.R_DOCKER_IMAGE, inputBindings, outputBinding, rParams, null); + } catch (Exception e) { + throw new ToolException(e); } + + // Check output file + if (!outputFile.exists()) { + throw new ToolException("Something wrong when computing the clustered values."); + } + + return outputFile; + } + + private String getClusteredKey(Variant variant) { + return NON_CLUSTERED; + } + + private String getTypeKey(Variant variant) { + String variantType = variant.getType() != null ? variant.getType().name() : ""; + if (CollectionUtils.isNotEmpty(variant.getStudies()) && CollectionUtils.isNotEmpty(variant.getStudies().get(0).getFiles())) { + for (FileEntry file : variant.getStudies().get(0).getFiles()) { + if (file.getData() != null) { + if (file.getData().containsKey("EXT_SVTYPE")) { + variantType = file.getData().get("EXT_SVTYPE").toUpperCase(Locale.ROOT); + break; + } else if (file.getData().containsKey("SVCLASS")) { + variantType = file.getData().get("SVCLASS").toUpperCase(Locale.ROOT); + break; + } + } + } + } + + switch (variantType) { + case "DEL": + case "DELETION": + return TYPE_DEL; + case "DUP": + case "TDS": + case "DUPLICATION": + case "TANDEM_DUPLICATION": + return TYPE_TDS; + case "INV": + case "INVERSION": + return TYPE_INV; + case "TR": + case "TRANS": + case "TRANSLOCATION": + return TYPE_TRANS; + } + return null; + } + + private String getLengthKey(Variant variant) { + if (variant.getSv() == null || variant.getSv().getBreakend() == null || variant.getSv().getBreakend().getMate() == null) { + return null; + } + BreakendMate mate = variant.getSv().getBreakend().getMate(); + if (variant.getChromosome().equals(mate.getChromosome())) { + int length = Math.abs(mate.getPosition() - variant.getStart()); + if (length <= 10000) { + return LENGTH_1_10Kb; + } else if (length <= 100000) { + return LENGTH_10Kb_100Kb; + } else if (length <= 1000000) { + return LENGTH_100Kb_1Mb; + } else if (length <= 10000000) { + return LENGTH_1Mb_10Mb; + } + return LENGTH_10Mb; + } else { + if (variant.getType() == VariantType.TRANSLOCATION) { + return LENGTH_NA; + } + } + return null; + } + + private void computeSignatureFitting() throws IOException, ToolException, CatalogException { + File cataloguesFile = getOutDir().resolve(CATALOGUES_FILENAME_DEFAULT).toFile(); + if (!cataloguesFile.exists()) { + // Get counts from sample + CatalogManager catalogManager = getVariantStorageManager().getCatalogManager(); + // Check sample + String study = catalogManager.getStudyManager().get(getStudy(), QueryOptions.empty(), getToken()).first().getFqn(); + OpenCGAResult sampleResult = catalogManager.getSampleManager().get(study, getSample(), QueryOptions.empty(), + getToken()); + if (sampleResult.getNumResults() != 1) { + throw new ToolException("Unable to compute mutational signature analysis. Sample '" + getSample() + "' not found"); + } + Sample sample = sampleResult.first(); + logger.info("Searching catalogue counts from quality control for sample " + getSample()); + if (sample.getQualityControl() != null && sample.getQualityControl().getVariant() != null + && CollectionUtils.isNotEmpty(sample.getQualityControl().getVariant().getSignatures())) { + logger.info("Searching in " + sample.getQualityControl().getVariant().getSignatures().size() + " signatures"); + for (Signature signature : sample.getQualityControl().getVariant().getSignatures()) { + logger.info("Matching ? " + getQueryId() + " vs " + signature.getId()); + if (getQueryId().equals(signature.getId())) { + // Write catalogue file + try (PrintWriter pw = new PrintWriter(cataloguesFile)) { + pw.println(getSample()); + for (Signature.GenomeContextCount count : signature.getCounts()) { + pw.println(count.getContext() + "\t" + count.getTotal()); + } + pw.close(); + } catch (Exception e) { + throw new ToolException("Error writing catalogue output file: " + cataloguesFile.getName(), e); + } + logger.info("Found catalogue {} and written in {}", signature.getId(), cataloguesFile.getAbsolutePath()); + break; + } + } + } + if (!cataloguesFile.exists()) { + throw new ToolException("Could not find mutational signagure catalogue (counts) file: " + cataloguesFile.getName()); + } + } + List> inputBindings = new ArrayList<>(); - inputBindings.add(new AbstractMap.SimpleEntry<>(inputPath, "/data/input")); + inputBindings.add(new AbstractMap.SimpleEntry<>(getOutDir().toAbsolutePath().toString(), "/data/input")); if (StringUtils.isNotEmpty(getSignaturesFile())) { File signaturesFile = new File(getSignaturesFile()); if (signaturesFile.exists()) { @@ -275,7 +553,7 @@ private void executeRScript() throws IOException { .toAbsolutePath().toString(), "/data/output"); StringBuilder scriptParams = new StringBuilder("R CMD Rscript --vanilla ") .append("/opt/opencga/signature.tools.lib/scripts/signatureFit") - .append(" --catalogues=/data/input/").append(new File(getCatalogues()).getName()) + .append(" --catalogues=/data/input/").append(cataloguesFile.getName()) .append(" --outdir=/data/output"); if (StringUtils.isNotEmpty(getFitMethod())) { scriptParams.append(" --fitmethod=").append(getFitMethod()); @@ -318,5 +596,16 @@ private void executeRScript() throws IOException { String cmdline = DockerUtils.run(R_DOCKER_IMAGE, inputBindings, outputBinding, scriptParams.toString(), null); logger.info("Docker command line: " + cmdline); + + // Check fitting file before parsing and creating the mutational signature fitting data model + File signatureCoeffsFile = getOutDir().resolve(SIGNATURE_COEFFS_FILENAME).toFile(); + if (!signatureCoeffsFile.exists()) { + throw new ToolExecutorException("Something wrong happened: signature coeffs. file " + SIGNATURE_COEFFS_FILENAME + " could not" + + " be generated"); + } + SignatureFitting signatureFitting = parseFittingResults(getOutDir(), getFitId(), getFitMethod(), getSigVersion(), getnBoot(), + getOrgan(), getThresholdPerc(), getThresholdPval(), getMaxRareSigs()); + JacksonUtils.getDefaultObjectMapper().writerFor(SignatureFitting.class).writeValue(getOutDir() + .resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_FITTING_DATA_MODEL_FILENAME).toFile(), signatureFitting); } } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java index a31c301ec86..a30c0b4f4b1 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/OpenCGATestExternalResource.java @@ -32,14 +32,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.text.SimpleDateFormat; import java.util.Date; @@ -171,6 +169,12 @@ public Path isolateOpenCGA() throws IOException { // Files.copy(inputStream, opencgaHome.resolve("examples") // .resolve("1k.chr1.phase3_shapeit2_mvncall_integrated_v5.20130502.genotypes.vcf.gz"), StandardCopyOption.REPLACE_EXISTING); + // Analysis + Files.createDirectories(opencgaHome.resolve("analysis/mutational-signature")); + + inputStream = new FileInputStream("../opencga-app/app/analysis/mutational-signature/sv_clustering.R"); + Files.copy(inputStream, opencgaHome.resolve("analysis/mutational-signature/sv_clustering.R"), StandardCopyOption.REPLACE_EXISTING); + return opencgaHome; } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java index 8d13a664edf..b2bd0a2227f 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java @@ -19,18 +19,19 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.mutable.MutableInt; import org.hamcrest.CoreMatchers; -import org.junit.AfterClass; -import org.junit.Assume; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.opencb.biodata.models.clinical.Disorder; import org.opencb.biodata.models.clinical.Phenotype; +import org.opencb.biodata.models.clinical.qc.HRDetect; import org.opencb.biodata.models.clinical.qc.SampleQcVariantStats; +import org.opencb.biodata.models.clinical.qc.Signature; +import org.opencb.biodata.models.clinical.qc.SignatureFitting; import org.opencb.biodata.models.core.SexOntologyTermAnnotation; import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; +import org.opencb.biodata.models.variant.avro.VariantType; import org.opencb.biodata.models.variant.metadata.SampleVariantStats; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; @@ -38,8 +39,10 @@ import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.analysis.tools.ToolRunner; import org.opencb.opencga.analysis.variant.gwas.GwasAnalysis; +import org.opencb.opencga.analysis.variant.hrdetect.HRDetectAnalysis; import org.opencb.opencga.analysis.variant.knockout.KnockoutAnalysis; import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.analysis.variant.mutationalSignature.MutationalSignatureAnalysis; import org.opencb.opencga.analysis.variant.operations.VariantIndexOperationTool; import org.opencb.opencga.analysis.variant.operations.VariantSampleIndexOperationTool; import org.opencb.opencga.analysis.variant.samples.SampleEligibilityAnalysis; @@ -68,16 +71,19 @@ import org.opencb.opencga.core.models.individual.Location; import org.opencb.opencga.core.models.operations.variant.VariantSampleIndexParams; import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.sample.SampleQualityControl; import org.opencb.opencga.core.models.sample.SampleReferenceParam; import org.opencb.opencga.core.models.sample.SampleUpdateParams; import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.variant.*; +import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.result.ExecutionResult; import org.opencb.opencga.core.tools.result.ExecutionResultManager; import org.opencb.opencga.storage.core.StorageEngineFactory; import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; +import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine; @@ -90,6 +96,8 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.net.URI; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; @@ -97,6 +105,7 @@ import static org.hamcrest.CoreMatchers.hasItem; import static org.junit.Assert.*; +import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.getResourceUri; @RunWith(Parameterized.class) public class VariantAnalysisTest { @@ -115,10 +124,15 @@ public class VariantAnalysisTest { private static String son = "NA19685"; private static String daughter = "NA19600"; + public static final String CANCER_STUDY = "cancer"; + private static String cancer_sample = "AR2.10039966-01T"; + private static String germline_sample = "AR2.10039966-01G"; + + @Parameterized.Parameters(name = "{0}") public static Object[][] parameters() { return new Object[][]{ - {MongoDBVariantStorageEngine.STORAGE_ENGINE_ID}, +// {MongoDBVariantStorageEngine.STORAGE_ENGINE_ID}, {HadoopVariantStorageEngine.STORAGE_ENGINE_ID} }; } @@ -172,13 +186,16 @@ public void setUp() throws Throwable { setUpCatalogManager(); - file = opencga.createFile(STUDY, "variant-test-file.vcf.gz", token); variantStorageManager.index(STUDY, file.getId(), opencga.createTmpOutdir("_index"), new ObjectMap(VariantStorageOptions.ANNOTATE.key(), true), token); for (int i = 0; i < file.getSampleIds().size(); i++) { + String id = file.getSampleIds().get(i); + if (id.equals(son)) { + SampleUpdateParams updateParams = new SampleUpdateParams().setSomatic(true); + catalogManager.getSampleManager().update(STUDY, id, updateParams, null, token); + } if (i % 2 == 0) { - String id = file.getSampleIds().get(i); SampleUpdateParams updateParams = new SampleUpdateParams().setPhenotypes(Collections.singletonList(PHENOTYPE)); catalogManager.getSampleManager().update(STUDY, id, updateParams, null, token); } @@ -217,6 +234,20 @@ public void setUp() throws Throwable { individuals.stream().map(Individual::getId).collect(Collectors.toList()), new QueryOptions(), token); + // Cancer (SV) + ObjectMap config = new ObjectMap(); +// config.put(VariantStorageOptions.ANNOTATE.key(), true); + config.put(VariantStorageOptions.LOAD_SPLIT_DATA.key(), VariantStorageEngine.SplitData.MULTI); + + file = opencga.createFile(CANCER_STUDY, "AR2.10039966-01T_vs_AR2.10039966-01G.annot.brass.vcf.gz", token); + variantStorageManager.index(CANCER_STUDY, file.getId(), opencga.createTmpOutdir("_index"), config, token); + file = opencga.createFile(CANCER_STUDY, "AR2.10039966-01T.copynumber.caveman.vcf.gz", token); + variantStorageManager.index(CANCER_STUDY, file.getId(), opencga.createTmpOutdir("_index"), config, token); + file = opencga.createFile(CANCER_STUDY, "AR2.10039966-01T_vs_AR2.10039966-01G.annot.pindel.vcf.gz", token); + variantStorageManager.index(CANCER_STUDY, file.getId(), opencga.createTmpOutdir("_index"), config, token); + + SampleUpdateParams updateParams = new SampleUpdateParams().setSomatic(true); + catalogManager.getSampleManager().update(CANCER_STUDY, cancer_sample, updateParams, null, token); opencga.getStorageConfiguration().getVariant().setDefaultEngine(storageEngine); VariantStorageEngine engine = opencga.getStorageEngineFactory().getVariantStorageEngine(storageEngine, DB_NAME); @@ -257,6 +288,19 @@ public void setUpCatalogManager() throws IOException, CatalogException { catalogManager.getSampleManager().create(STUDY, sample, null, token); } + // Cancer + List samples = new ArrayList<>(); + catalogManager.getStudyManager().create(projectId, CANCER_STUDY, null, "Phase 1", "Done", null, null, null, null, null, token); + Sample sample = new Sample().setId(cancer_sample).setSomatic(true); + samples.add(sample); +// catalogManager.getSampleManager().create(CANCER_STUDY, sample, null, token); + sample = new Sample().setId(germline_sample); + samples.add(sample); +// catalogManager.getSampleManager().create(CANCER_STUDY, sample, null, token); + Individual individual = catalogManager.getIndividualManager() + .create(CANCER_STUDY, new Individual("AR2.10039966-01", "AR2.10039966-01", new Individual(), new Individual(), new Location(), SexOntologyTermAnnotation.initMale(), null, null, null, null, "", + samples, false, 0, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), Collections.emptyMap()), Collections.emptyList(), new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), token).first(); + assertEquals(2, individual.getSamples().size()); } @Test @@ -745,7 +789,289 @@ public void testVariantSecondarySampleIndex() throws Exception { // checkExecutionResult(er, false); } - public void checkExecutionResult(ExecutionResult er) { + @Test + public void testMutationalSignatureFittingSNV() throws Exception { + Path outDir = Paths.get(opencga.createTmpOutdir("_mutational_signature_fitting_snv")); + System.out.println("outDir = " + outDir); + + URI uri = getResourceUri("mutational-signature-catalogue-snv.json"); + Path path = Paths.get(uri.getPath()); + Signature signature = JacksonUtils.getDefaultObjectMapper().readerFor(Signature.class).readValue(path.toFile()); + SampleQualityControl qc = new SampleQualityControl(); + qc.getVariant().setSignatures(Collections.singletonList(signature)); + SampleUpdateParams updateParams = new SampleUpdateParams().setQualityControl(qc); + catalogManager.getSampleManager().update(CANCER_STUDY, cancer_sample, updateParams, null, token); + + MutationalSignatureAnalysisParams params = new MutationalSignatureAnalysisParams(); + params.setSample(cancer_sample); + params.setId(signature.getId()); + params.setFitId("fitting-1"); + params.setFitMethod("FitMS"); + params.setFitSigVersion("RefSigv2"); + params.setFitOrgan("Breast"); + params.setFitNBoot(200); + params.setFitThresholdPerc(5.0f); + params.setFitThresholdPval(0.05f); + params.setFitMaxRareSigs(1); + params.setSkip("catalogue"); + + toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), + outDir, null, token); + + java.io.File catalogueFile = outDir.resolve(MutationalSignatureAnalysis.SIGNATURE_COEFFS_FILENAME).toFile(); + byte[] bytes = Files.readAllBytes(catalogueFile.toPath()); + System.out.println(new String(bytes)); + assertTrue(catalogueFile.exists()); + + java.io.File signatureFile = outDir.resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_FITTING_DATA_MODEL_FILENAME).toFile(); + bytes = Files.readAllBytes(signatureFile.toPath()); + System.out.println(new String(bytes)); + assertTrue(signatureFile.exists()); + + OpenCGAResult sampleResult = catalogManager.getSampleManager().get(CANCER_STUDY, cancer_sample, QueryOptions.empty(), token); + Sample sample = sampleResult.first(); + List signatures = sample.getQualityControl().getVariant().getSignatures(); + for (Signature sig : signatures) { + if (sig.getId().equals(signature.getId())) { + for (SignatureFitting fitting : sig.getFittings()) { + if (fitting.getId().equals(params.getFitId())) { + System.out.println(JacksonUtils.getDefaultObjectMapper().writerFor(SignatureFitting.class).writeValueAsString(fitting)); + return; + } + } + } + } + fail("Mutational signature fitting not found in sample quality control"); + } + + @Test + public void testMutationalSignatureCatalogueSV() throws Exception { + Path outDir = Paths.get(opencga.createTmpOutdir("_mutational_signature_catalogue_sv")); + System.out.println("outDir = " + outDir); + + Path opencgaHome = opencga.getOpencgaHome(); + System.out.println("OpenCGA home = " + opencgaHome); + + MutationalSignatureAnalysisParams params = new MutationalSignatureAnalysisParams(); + params.setSample(cancer_sample); + params.setId("catalogue-1"); + params.setDescription("Catalogue #1"); + VariantQuery query = new VariantQuery(); + query.sample(cancer_sample); + query.type(VariantType.SV.name()); + params.setQuery(query.toJson()); + params.setSkip("fitting"); + + toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), + outDir, null, token); + + java.io.File catalogueFile = outDir.resolve(MutationalSignatureAnalysis.CATALOGUES_FILENAME_DEFAULT).toFile(); + byte[] bytes = Files.readAllBytes(catalogueFile.toPath()); + System.out.println(new String(bytes)); + assertTrue(catalogueFile.exists()); + + java.io.File signatureFile = outDir.resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_DATA_MODEL_FILENAME).toFile(); + bytes = Files.readAllBytes(signatureFile.toPath()); + System.out.println(new String(bytes)); + assertTrue(signatureFile.exists()); + + OpenCGAResult sampleResult = catalogManager.getSampleManager().get(CANCER_STUDY, cancer_sample, QueryOptions.empty(), token); + Sample sample = sampleResult.first(); + List signatures = sample.getQualityControl().getVariant().getSignatures(); + for (Signature signature : signatures) { + if (signature.getId().equals(params.getId())) { + return; + } + } + fail("Signature not found in sample quality control"); + } + + @Test + public void testMutationalSignatureFittingSV() throws Exception { + Path outDir = Paths.get(opencga.createTmpOutdir("_mutational_signature_fitting")); + System.out.println("outDir = " + outDir); + + URI uri = getResourceUri("2019_01_10_all_PCAWG_sigs_rearr.tsv"); + Path path = Paths.get(uri.getPath()); + catalogManager.getFileManager().createFolder(CANCER_STUDY, "signature", true, "", new QueryOptions(), token); + catalogManager.getFileManager().link(CANCER_STUDY, uri, "signature", new ObjectMap(), token); + String filename = Paths.get(uri.toURL().getFile()).toFile().getName(); + File file = catalogManager.getFileManager().get(CANCER_STUDY, filename, null, token).first(); + String signatureFileId = file.getId(); + + uri = getResourceUri("mutational-signature-sv.json"); + path = Paths.get(uri.getPath()); + Signature signature = JacksonUtils.getDefaultObjectMapper().readerFor(Signature.class).readValue(path.toFile()); + SampleQualityControl qc = new SampleQualityControl(); + qc.getVariant().setSignatures(Collections.singletonList(signature)); + SampleUpdateParams updateParams = new SampleUpdateParams().setQualityControl(qc); + catalogManager.getSampleManager().update(CANCER_STUDY, cancer_sample, updateParams, null, token); + + MutationalSignatureAnalysisParams params = new MutationalSignatureAnalysisParams(); + params.setSample(cancer_sample); + params.setId(signature.getId()); + params.setFitId("fitting-1"); + params.setFitMethod("FitMS"); + params.setFitSigVersion("RefSigv2"); + params.setFitOrgan("Breast"); + params.setFitNBoot(200); + params.setFitThresholdPerc(5.0f); + params.setFitThresholdPval(0.05f); + params.setFitMaxRareSigs(1); + params.setFitSignaturesFile(signatureFileId); + params.setFitRareSignaturesFile(signatureFileId); + params.setSkip("catalogue"); + + toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), + outDir, null, token); + + java.io.File catalogueFile = outDir.resolve(MutationalSignatureAnalysis.SIGNATURE_COEFFS_FILENAME).toFile(); + byte[] bytes = Files.readAllBytes(catalogueFile.toPath()); + System.out.println(new String(bytes)); + assertTrue(catalogueFile.exists()); + + java.io.File signatureFile = outDir.resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_FITTING_DATA_MODEL_FILENAME).toFile(); + bytes = Files.readAllBytes(signatureFile.toPath()); + System.out.println(new String(bytes)); + assertTrue(signatureFile.exists()); + } + + @Test + public void testHRDetect() throws Exception { + Path snvFittingOutDir = Paths.get(opencga.createTmpOutdir("_snv_fitting")); + Path svFittingOutDir = Paths.get(opencga.createTmpOutdir("_sv_fitting")); + Path hrdetectOutDir = Paths.get(opencga.createTmpOutdir("_hrdetect")); + + // Read SNV signaure + URI uri = getResourceUri("mutational-signature-catalogue-snv.json"); + Path path = Paths.get(uri.getPath()); + Signature snvSignature = JacksonUtils.getDefaultObjectMapper().readerFor(Signature.class).readValue(path.toFile()); + + // Read SV signature + uri = getResourceUri("mutational-signature-sv.json"); + path = Paths.get(uri.getPath()); + Signature svSignature = JacksonUtils.getDefaultObjectMapper().readerFor(Signature.class).readValue(path.toFile()); + + // Update quality control for the cancer sample + SampleQualityControl qc = new SampleQualityControl(); + qc.getVariant().setSignatures(Arrays.asList(snvSignature, svSignature)); + SampleUpdateParams updateParams = new SampleUpdateParams().setQualityControl(qc); + catalogManager.getSampleManager().update(CANCER_STUDY, cancer_sample, updateParams, null, token); + + // SNV fitting + MutationalSignatureAnalysisParams params = new MutationalSignatureAnalysisParams(); + params.setSample(cancer_sample); + params.setId(snvSignature.getId()); + params.setFitId("snv-fitting-1"); + params.setFitMethod("FitMS"); + params.setFitSigVersion("RefSigv2"); + params.setFitOrgan("Breast"); + params.setFitNBoot(100); + params.setFitThresholdPerc(5.0f); + params.setFitThresholdPval(0.05f); + params.setFitMaxRareSigs(1); + params.setSkip("catalogue"); + + toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), + snvFittingOutDir, null, token); + + java.io.File snvSignatureFittingFile = snvFittingOutDir.resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_FITTING_DATA_MODEL_FILENAME).toFile(); + assertTrue(snvSignatureFittingFile.exists()); + SignatureFitting snvFitting = JacksonUtils.getDefaultObjectMapper().readerFor(SignatureFitting.class).readValue(snvSignatureFittingFile); + assertEquals(params.getFitId(), snvFitting.getId()); + + // SV fitting + uri = getResourceUri("2019_01_10_all_PCAWG_sigs_rearr.tsv"); + path = Paths.get(uri.getPath()); + catalogManager.getFileManager().createFolder(CANCER_STUDY, "signature", true, "", new QueryOptions(), token); + catalogManager.getFileManager().link(CANCER_STUDY, uri, "signature", new ObjectMap(), token); + String filename = Paths.get(uri.toURL().getFile()).toFile().getName(); + File file = catalogManager.getFileManager().get(CANCER_STUDY, filename, null, token).first(); + String signatureFileId = file.getId(); + + params = new MutationalSignatureAnalysisParams(); + params.setSample(cancer_sample); + params.setId(svSignature.getId()); + params.setFitId("fitting-sv-1"); + params.setFitMethod("FitMS"); + params.setFitSigVersion("RefSigv2"); + params.setFitOrgan("Breast"); + params.setFitNBoot(100); + params.setFitThresholdPerc(5.0f); + params.setFitThresholdPval(0.05f); + params.setFitMaxRareSigs(1); + params.setFitSignaturesFile(signatureFileId); + params.setFitRareSignaturesFile(signatureFileId); + params.setSkip("catalogue"); + + toolRunner.execute(MutationalSignatureAnalysis.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), + svFittingOutDir, null, token); + + java.io.File svSignatureFittingFile = svFittingOutDir.resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_FITTING_DATA_MODEL_FILENAME).toFile(); + assertTrue(svSignatureFittingFile.exists()); + SignatureFitting svFitting = JacksonUtils.getDefaultObjectMapper().readerFor(SignatureFitting.class).readValue(svSignatureFittingFile); + assertEquals(params.getFitId(), svFitting.getId()); + + // HRDetect + HRDetectAnalysisParams hrdParams = new HRDetectAnalysisParams(); + hrdParams.setId("hrd-1"); + hrdParams.setSampleId(cancer_sample); + hrdParams.setSnvFittingId(snvFitting.getId()); + hrdParams.setSvFittingId(svFitting.getId()); + hrdParams.setCnvQuery("{\"sample\": \"" + cancer_sample + "\", \"type\": \"" + VariantType.CNV + "\"}"); + hrdParams.setIndelQuery("{\"sample\": \"" + cancer_sample + "\", \"type\": \"" + VariantType.INDEL + "\"}"); + hrdParams.setBootstrap(true); + + toolRunner.execute(HRDetectAnalysis.class, hrdParams, new ObjectMap(ParamConstants.STUDY_PARAM, CANCER_STUDY), hrdetectOutDir, null, token); + + java.io.File hrDetectFile = hrdetectOutDir.resolve(HRDetectAnalysis.HRDETECT_SCORES_FILENAME_DEFAULT).toFile(); + byte[] bytes = Files.readAllBytes(hrDetectFile.toPath()); + System.out.println(new String(bytes)); + assertTrue(hrDetectFile.exists()); + + OpenCGAResult sampleResult = catalogManager.getSampleManager().get(CANCER_STUDY, cancer_sample, QueryOptions.empty(), token); + Sample sample = sampleResult.first(); + List hrDetects = sample.getQualityControl().getVariant().getHrDetects(); + for (HRDetect hrDetect : hrDetects) { + if (hrDetect.getId().equals(hrDetect.getId())) { + if (hrDetect.getScores().containsKey("del.mh.prop")) { + Assert.assertEquals(hrDetect.getScores().getFloat("del.mh.prop"), 0.172413793103448f, 0.00001f); + return; + } + } + } + fail("HRDetect result not found in sample quality control"); + } + + @Test + public void testHRDetectParseResults() throws Exception { + Path hrdetectOutDir = Paths.get(opencga.createTmpOutdir("_hrdetect")); + URI uri = getResourceUri("hrdetect_output_38.tsv"); + java.io.File file = Paths.get(uri.getPath()).toFile(); + FileUtils.copyFile(file, hrdetectOutDir.resolve(HRDetectAnalysis.HRDETECT_SCORES_FILENAME_DEFAULT).toFile()); + + HRDetectAnalysisParams hrdParams = new HRDetectAnalysisParams(); + hrdParams.setId("hrd-1"); + hrdParams.setSampleId(cancer_sample); + hrdParams.setSnvFittingId("snvFittingId"); + hrdParams.setSvFittingId("svFittingId"); + hrdParams.setCnvQuery("{\"sample\": \"" + cancer_sample + "\", \"type\": \"" + VariantType.CNV + "\"}"); + hrdParams.setIndelQuery("{\"sample\": \"" + cancer_sample + "\", \"type\": \"" + VariantType.INDEL + "\"}"); + + HRDetectAnalysis analysis = new HRDetectAnalysis(); + analysis.setUp(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager, hrdParams.toObjectMap(), hrdetectOutDir, + "job-1", token); + HRDetect hrDetect = analysis.parseResult(hrdetectOutDir); + for (Map.Entry entry : hrDetect.getScores().entrySet()) { + System.out.println(entry.getKey() + " -> " + entry.getValue()); + } + assertTrue(hrDetect.getScores().containsKey("hrd")); + assertEquals(-0.102769986f, hrDetect.getScores().getFloat("hrd"), 0.00001f); + assertTrue(hrDetect.getScores().containsKey("Probability")); + assertEquals(0.998444f, hrDetect.getScores().getFloat("Probability"), 0.00001f); + } + + public void checkExecutionResult(ExecutionResult er) { checkExecutionResult(er, true); } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java index b789e5b96c7..c03587a1011 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java @@ -188,7 +188,7 @@ public static void setUp() throws Exception { .setVersion("v5") .setDefaultSpecies("hsapiens") .setRest(new RestConfig(Collections.singletonList("https://ws.zettagenomics.com/cellbase"), 10000)); - cellBaseUtils = new CellBaseUtils(new CellBaseClient(clientConfiguration), assembly); + cellBaseUtils = new CellBaseUtils(new CellBaseClient(clientConfiguration)); Region cadm1 = cellBaseUtils.getGeneRegion("CADM1"); diff --git a/opencga-app/app/analysis/mutational-signature/sv_clustering.R b/opencga-app/app/analysis/mutational-signature/sv_clustering.R new file mode 100644 index 00000000000..83f5fe2d459 --- /dev/null +++ b/opencga-app/app/analysis/mutational-signature/sv_clustering.R @@ -0,0 +1,884 @@ +library(optparse) + +#' The BEDPE data fram should contain the following columns: "chrom1", "start1", "end1", "chrom2", "start2", "end2" and "sample" (sample name). + +clustering <- function(sv_bedpe, + out_fpath, + kmin, + kmin.samples, + gamma.sdev, + PEAK.FACTOR, + thresh.dist, + gamma, + kmin.filter) { + sv_bedpe <- read.table(args[1], sep = "\t", header = TRUE, stringsAsFactors = FALSE, check.names = FALSE) + clustering.result <- rearrangement.clustering_bedpe(sv_bedpe=sv_bedpe, + kmin=kmin, + kmin.samples=kmin.samples, + gamma.sdev=gamma.sdev, + PEAK.FACTOR=PEAK.FACTOR, + thresh.dist=thresh.dist, + gamma=gamma, + kmin.filter=kmin.filter) + sv_bedpe <- clustering.result$sv_bedpe + write.table(sv_bedpe, file = out_fpath, row.names = FALSE, sep = "\t", quote = FALSE) +} + +calcIntermutDist <- function (subs.type, first.chrom.na = FALSE) { + + subs.type.processed <- data.frame() + for (c in unique(subs.type$chr)) { + # choose subs from only one chromosome at a time + + subs.type.chrom <- subset(subs.type, subset=subs.type$chr==c) + # sort the subs by position + subs.type.chrom <- subs.type.chrom [order(subs.type.chrom$position),] + + if (first.chrom.na) { + subs.type.chrom$prevPos <- c(NA,subs.type.chrom$position[1:nrow(subs.type.chrom)-1]) + } else { + subs.type.chrom$prevPos <- c(0,subs.type.chrom$position[1:nrow(subs.type.chrom)-1]) + } + subs.type.chrom$distPrev <- subs.type.chrom$position - subs.type.chrom$prevPos + + subs.type.processed <- rbind(subs.type.processed,subs.type.chrom) + } + + subs.type.processed$distPrev[subs.type.processed$distPrev==0] <- 1 + subs.type.processed +} + +assignPvalues <- function(kat.regions, chrom.bps, bp.rate=NA) { + + if (is.na(bp.rate)) { # estimate the chromosome rate + left.bp <- min(chrom.bps$pos) + right.bp <- max(chrom.bps$pos) + bp.rate <- nrow(chrom.bps)/ (right.bp - left.bp) + } + + # assume binomial distribution + kat.regions$pvalue <- 1-pbinom(kat.regions$number.bps, kat.regions$end.bp - kat.regions$start.bp, bp.rate) + + kat.regions$d.seg<- (kat.regions$number.bps/( kat.regions$end.bp - kat.regions$start.bp)) + + kat.regions$rate.factor <- kat.regions$d.seg/bp.rate + + kat.regions +} + +hotspotInfo <- function(kat.regions.all, subs, segInterDist=c()) { + if(nrow(kat.regions.all)>0){ + for(r in 1:nrow(kat.regions.all)){ + + # indices of the breakpoints in the hotspot + subs.hotspot <-subs[kat.regions.all$firstBp[r]:kat.regions.all$lastBp[r],] + + kat.regions.all[r,'start.bp'] <- min(subs.hotspot$pos) + kat.regions.all[r,'end.bp'] <- max(subs.hotspot$pos) + kat.regions.all[r,'length.bp'] <- kat.regions.all[r,'end.bp'] - kat.regions.all[r,'start.bp'] + kat.regions.all[r,'number.bps'] <- nrow(subs.hotspot) + kat.regions.all[r,'number.bps.clustered'] <- sum(subs.hotspot$is.clustered) + + if (length(segInterDist)>0 & is.na(kat.regions.all[r,'avgDist.bp'])) { + kat.regions.all[r,'avgDist.bp'] <- mean(segInterDist[kat.regions.all$firstBp[r]:kat.regions.all$lastBp[r]]) + } + kat.regions.all[r,'no.samples'] <- length(unique(subs.hotspot$sample)) + + if ('pf' %in% colnames(subs.hotspot)){ + kat.regions.all[r,'no.del'] <- nrow(subset(subs.hotspot, pf==2)) + kat.regions.all[r,'no.dup'] <- nrow(subset(subs.hotspot, pf==4)) + kat.regions.all[r,'no.inv'] <- nrow(subset(subs.hotspot, pf==1 | pf==8)) + kat.regions.all[r,'no.trn'] <- nrow(subset(subs.hotspot, pf==32)) + } + + } # for all peaks + } # if there is at least one peak + kat.regions.all +} + +extract.kat.regions <- function (res, imd, subs, kmin.samples=10, pvalue.thresh=1, rate.factor.thresh=1, doMerging=FALSE, kmin.filter=NA, bp.rate=NA) { + + segInterDist <- res$yhat + kataegis.threshold <- imd + + kat.regions.all = data.frame() + + chr <- as.character(subs$chr[1]) + + positions <- subs$pos + + katLoci = (segInterDist<=kataegis.threshold) # flag specifying if a point is in a peak + + if(sum(katLoci)>0) { + + start.regions = which(katLoci[-1] & !(katLoci[-(length(katLoci))]) # katLoci breakpoints + | (katLoci[-1] & katLoci[-(length(katLoci))] & segInterDist[-1] != segInterDist[-length(katLoci)] ) + )+1 # endpoints between peaks + if (katLoci[1]) {start.regions <- c(1, start.regions)} + + end.regions = which(!(katLoci[-1]) & katLoci[-(length(katLoci))] # + | (katLoci[-1] & katLoci[-(length(katLoci))] & segInterDist[-1] != segInterDist[-length(katLoci)] ) + ) # + if (katLoci[length(katLoci)]) {end.regions <- c( end.regions, length(katLoci))} + + start.regions.init <- start.regions + end.regions.init <- end.regions + + # handling special cases + if(length(end.regions)+length(start.regions)>0) { # if there are any discontinuities in the segmentation at all + if (length(end.regions)==1 & length(start.regions)==0){ + start.regions <- 1 + } else if (length(start.regions)==1 & length(end.regions)==0){ + end.regions <- length(positions) + } else if ((end.regions[1]end.regions[length(end.regions)])) { + # starts and ends are the same length, but missing both endpoints + + start.regions <- c(1,start.regions) + end.regions <- c(end.regions, length(positions)) + + } else if (end.regions[1]end.regions[length(end.regions)]){ + # ends will be one shorter + + end.regions <- c(end.regions, length(positions)) + } + + if (length(start.regions)!=length(end.regions)) { + browser() + } + + + + # prepare a data structure that will be later filled up + kat.regions.all <- data.frame( + chr=subs$chr[1], + start.bp=rep(NA,length(start.regions)), # start coordinate [bp] + end.bp=rep(NA,length(start.regions)), # end coordinate [bp] + length.bp=rep(NA,length(start.regions)), # length [bp] + number.bps=rep(NA,length(start.regions)), + number.bps.clustered=rep(NA,length(start.regions)), + avgDist.bp=rep(NA,length(start.regions)), + no.samples=rep(NA,length(start.regions)), + no.del =rep(NA,length(start.regions)), + no.dup =rep(NA,length(start.regions)), + no.inv= rep(NA,length(start.regions)), + no.trn = rep(NA,length(start.regions)), + firstBp=start.regions, + lastBp=end.regions ) + + kat.regions.all <- hotspotInfo(kat.regions.all, subs, segInterDist) + + step.segInterDist.left <- rep(NA, length(segInterDist)) + step.segInterDist.left[2:length(segInterDist)] <- segInterDist[2:length(segInterDist)]- segInterDist[1:(length(segInterDist)-1)] + step.segInterDist.right <- rep(NA, length(segInterDist)) + step.segInterDist.right[1:(length(segInterDist)-1)] <- segInterDist[1:(length(segInterDist)-1)]- segInterDist[2:(length(segInterDist))] + + kat.regions.all$step.left <- step.segInterDist.left[start.regions] + kat.regions.all$step.right <- step.segInterDist.right[end.regions] + + + # run the filters on the regions of increased frequency + # make sure there are at least kmin samples + + if ((!is.null(kat.regions.all)) && (nrow(kat.regions.all)>0)) { + kat.regions.all <- subset(kat.regions.all, no.samples>=kmin.samples) + } + + + # make sure there are at least kmin.filter breakpoints + if (!is.na(kmin.filter)) { + kat.regions.all <- subset(kat.regions.all, number.bps>=kmin.filter) + } + + + + # make sure the p-value is less than somethng + if ((!is.null(kat.regions.all)) && (nrow(kat.regions.all)>0)) { + kat.regions.all <- assignPvalues(kat.regions.all, subs, bp.rate=bp.rate) + kat.regions.all <- subset(kat.regions.all, pvalue<=pvalue.thresh) + # only keep the hotspots that exceed the theshold + kat.regions.all <- subset(kat.regions.all, rate.factor>=rate.factor.thresh) + } + + + + + + # merge segments if both were found to be peaks + if (doMerging) { + if(nrow(kat.regions.all)>1){ + for(r in 2:nrow(kat.regions.all)){ + if (kat.regions.all$lastBp[r-1] == (kat.regions.all$firstBp[r]-1)) { + # merge two segments + kat.regions.all$firstBp[r] <- kat.regions.all$firstBp[r-1] + kat.regions.all$firstBp[r-1] <- NA + kat.regions.all$lastBp[r-1] <- NA + kat.regions.all$avgDist.bp[r] <- NA # this will need to be updated as segments are being merged + } + } + } + # remove some of the merged segments + kat.regions.all <- subset(kat.regions.all, !is.na(firstBp) & !is.na(lastBp)) + + # update the info on hotspots that might have changed when they were merged + kat.regions.all <- hotspotInfo( kat.regions.all , subs, segInterDist) + kat.regions.all <- assignPvalues(kat.regions.all, subs, bp.rate=bp.rate) + } # end merging + + + + + } # end if there are discontinuities in the segmentation + } # if there are any points under the inter-mutation distance threshold + + kat.regions.all + +} + +#PCF-ALGORITHM (KL): +### EXACT version +exactPcf <- function(y, kmin=5, gamma, yest) { + ## Implementaion of exact PCF by Potts-filtering + ## x: input array of (log2) copy numbers + ## kmin: Mininal length of plateaus + ## gamma: penalty for each discontinuity + N <- length(y) + yhat <- rep(0,N); + if (N < 2*kmin) { + if (yest) { + return(list(Lengde = N, sta = 1, mean = mean(y), nIntervals=1, yhat=rep(mean(y),N))) + } else { + return(list(Lengde = N, sta = 1, mean = mean(y), nIntervals=1)) + } + } + initSum <- sum(y[1:kmin]) + initKvad <- sum(y[1:kmin]^2) + initAve <- initSum/kmin; + bestCost <- rep(0,N) + bestCost[kmin] <- initKvad - initSum*initAve + bestSplit <- rep(0,N) + bestAver <- rep(0,N) + bestAver[kmin] <- initAve + Sum <- rep(0,N) + Kvad <- rep(0,N) + Aver <- rep(0,N) + Cost <- rep(0,N) + kminP1=kmin+1 + for (k in (kminP1):(2*kmin-1)) { + Sum[kminP1:k]<-Sum[kminP1:k]+y[k] + Aver[kminP1:k] <- Sum[kminP1:k]/((k-kmin):1) + Kvad[kminP1:k] <- Kvad[kminP1:k]+y[k]^2 + bestAver[k] <- (initSum+Sum[kminP1])/k + bestCost[k] <- (initKvad+Kvad[kminP1])-k*bestAver[k]^2 + } + for (n in (2*kmin):N) { + yn <- y[n] + yn2 <- yn^2 + Sum[kminP1:n] <- Sum[kminP1:n]+yn + Aver[kminP1:n] <- Sum[kminP1:n]/((n-kmin):1) + Kvad[kminP1:n] <- Kvad[kminP1:n]+yn2 + nMkminP1=n-kmin+1 + Cost[kminP1:nMkminP1] <- bestCost[kmin:(n-kmin)]+Kvad[kminP1:nMkminP1]-Sum[kminP1:nMkminP1]*Aver[kminP1:nMkminP1]+gamma + Pos <- which.min(Cost[kminP1:nMkminP1])+kmin + cost <- Cost[Pos] + aver <- Aver[Pos] + totAver <- (Sum[kminP1]+initSum)/n + totCost <- (Kvad[kminP1]+initKvad) - n*totAver*totAver + + if (length(totCost)==0 || length(cost)==0) { + browser() + } + if (totCost < cost) { + Pos <- 1 + cost <- totCost + aver <- totAver + } + bestCost[n] <- cost + bestAver[n] <- aver + bestSplit[n] <- Pos-1 + } + n <- N + antInt <- 0 + if(yest){ + while (n > 0) { + yhat[(bestSplit[n]+1):n] <- bestAver[n] + n <- bestSplit[n] + antInt <- antInt+1 + } + } else { + while (n > 0) { + n <- bestSplit[n] + antInt <- antInt+1 + } + } + n <- N #nProbes + lengde <- rep(0,antInt) + start <- rep(0,antInt) + verdi <- rep(0,antInt) + oldSplit <- n + antall <- antInt + while (n > 0) { + start[antall] <- bestSplit[n]+1 + lengde[antall] <- oldSplit-bestSplit[n] + verdi[antall] <- bestAver[n] + n <- bestSplit[n] + oldSplit <- n + antall <- antall-1 + } + if (yest) { + return(list(Lengde = lengde, sta = start, mean = verdi, nIntervals=antInt, yhat=yhat)) + } else { + return(list(Lengde = lengde, sta = start, mean = verdi, nIntervals=antInt)) + } +} + + + +selectFastPcf <- function(x,kmin,gamma,yest){ + xLength <- length(x) + if (xLength< 1000) { + result<-runFastPcf(x,kmin,gamma,0.15,0.15,yest) + } else { + if (xLength < 15000){ + result<-runFastPcf(x,kmin,gamma,0.12,0.05,yest) + } else { + result<-runPcfSubset(x,kmin,gamma,0.12,0.05,yest) + } + } + return(result) +} + + +runFastPcf <- function(x,kmin,gamma,frac1,frac2,yest){ + antGen <- length(x) + + L <- min(8, floor(length(x)/6)) + + mark<-filterMarkS4(x,kmin,L,1,frac1,frac2,0.02,0.9) + mark[antGen]=TRUE + dense <- compact(x,mark) + #print(dense$Nr) + #print(frac2) + result<-PottsCompact(kmin,gamma,dense$Nr,dense$Sum,dense$Sq,yest) + return(result) +} + +runPcfSubset <- function(x,kmin,gamma,frac1,frac2,yest){ + SUBSIZE <- 5000 + antGen <- length(x) + mark<-filterMarkS4(x,kmin,8,1,frac1,frac2,0.02,0.9) + markInit<-c(mark[1:(SUBSIZE-1)],TRUE) + compX<-compact(x[1:SUBSIZE],markInit) + mark2 <- rep(FALSE,antGen) + mark2[1:SUBSIZE] <- markWithPotts(kmin,gamma,compX$Nr,compX$Sum,compX$Sq,SUBSIZE) + mark2[4*SUBSIZE/5]<-TRUE + start <- 4*SUBSIZE/5+1 + while(start + SUBSIZE < antGen){ + slutt<-start+SUBSIZE-1 + markSub<-c(mark2[1:(start-1)],mark[start:slutt]) + markSub[slutt] <- TRUE + compX<-compact(x[1:slutt],markSub) + mark2[1:slutt] <- markWithPotts(kmin,gamma,compX$Nr,compX$Sum,compX$Sq,slutt) + start <- start+4*SUBSIZE/5 + mark2[start-1]<-TRUE + } + markSub<-c(mark2[1:(start-1)],mark[start:antGen]) + compX<-compact(x,markSub) + result <- PottsCompact(kmin,gamma,compX$Nr,compX$Sum,compX$Sq,yest) + return(result) +} + +PottsCompact <- function(kmin, gamma, nr, res, sq, yest) { + ## Potts filtering on compact array; + ## kmin: minimal length of plateau + ## gamma: penalty for discontinuity + ## nr: number of values between breakpoints + ## res: sum of values between breakpoints + ## sq: sum of squares of values between breakpoints + + N <- length(nr) + Ant <- rep(0,N) + Sum <- rep(0,N) + Kvad <- rep(0,N) + Cost <- rep(0,N) + if (sum(nr) < 2*kmin){ + estim <- list() + estim$yhat <- rep( sum(res)/sum(nr),sum(nr)) + return(estim) + } + initAnt <- nr[1] + initSum <- res[1] + initKvad <- sq[1] + initAve <- initSum/initAnt + bestCost <- rep(0,N) + bestCost[1] <- initKvad - initSum*initAve + bestSplit <- rep(0,N) + k <- 2 + while(sum(nr[1:k]) < 2*kmin) { + Ant[2:k] <- Ant[2:k]+nr[k] + Sum[2:k]<-Sum[2:k]+res[k] + Kvad[2:k] <- Kvad[2:k]+sq[k] + bestCost[k] <- (initKvad+Kvad[2])-(initSum+Sum[2])^2/(initAnt+Ant[2]) + k <- k+1 + } + for (n in k:N) { + Ant[2:n] <- Ant[2:n]+nr[n] + Sum[2:n] <- Sum[2:n]+res[n] + Kvad[2:n] <- Kvad[2:n]+sq[n] + limit <- n + while(limit > 2 & Ant[limit] < kmin) {limit <- limit-1} + Cost[2:limit] <- bestCost[1:limit-1]+Kvad[2:limit]-Sum[2:limit]^2/Ant[2:limit] + Pos <- which.min(Cost[2:limit])+ 1 + cost <- Cost[Pos]+gamma + totCost <- (Kvad[2]+initKvad) - (Sum[2]+initSum)^2/(Ant[2]+initAnt) + if (totCost < cost) { + Pos <- 1 + cost <- totCost + } + bestCost[n] <- cost + bestSplit[n] <- Pos-1 + } + + if (yest) { + yhat<-rep(0,N) + res<-findEst(bestSplit,N,nr,res,TRUE) + } else { + res<-findEst(bestSplit,N,nr,res,FALSE) + } + return(res) +} + +compact <- function(y,mark){ + ## accumulates numbers of observations, sums and + ## sums of squares between potential breakpoints + N <- length(y) + tell<-seq(1:N) + cCTell<-tell[mark] + Ncomp<-length(cCTell) + lowTell<-c(0,cCTell[1:(Ncomp-1)]) + ant<-cCTell-lowTell + cy<-cumsum(y) + cCcy<-cy[mark] + lowcy<-c(0,cCcy[1:(Ncomp-1)]) + sum<-cCcy-lowcy + y2<-y^2 + cy2<-cumsum(y2) + cCcy2<-cy2[mark] + lowcy2<-c(0,cCcy2[1:(Ncomp-1)]) + sq<-cCcy2-lowcy2 + return(list(Nr=ant,Sum=sum,Sq=sq)) +} + +findEst <- function(bestSplit,N,Nr,Sum,yest){ + n<-N + lengde<-rep(0,N) + antInt<-0 + while (n>0){ + antInt<-antInt+1 + lengde[antInt] <- n-bestSplit[n] + n<-bestSplit[n] + } + lengde<-lengde[antInt:1] + lengdeOrig<-rep(0,antInt) + startOrig<-rep(1,antInt+1) + verdi<-rep(0,antInt) + start<-rep(1,antInt+1) + for(i in 1:antInt){ + start[i+1] <- start[i]+lengde[i] + lengdeOrig[i] <- sum(Nr[start[i]:(start[i+1]-1)]) + startOrig[i+1] <- startOrig[i]+lengdeOrig[i] + verdi[i]<-sum(Sum[start[i]:(start[i+1]-1)])/lengdeOrig[i] + } + + if(yest){ + yhat<-rep(0,startOrig[antInt+1]-1) + for (i in 1:antInt){ + yhat[startOrig[i]:(startOrig[i+1]-1)]<-verdi[i] + } + startOrig<-startOrig[1:antInt] + return(list(Lengde=lengdeOrig,sta=startOrig,mean=verdi,nIntervals=antInt,yhat=yhat)) + } else { + startOrig<-startOrig[1:antInt] + return(list(Lengde=lengdeOrig,sta=startOrig,mean=verdi,nIntervals=antInt)) + } + +} + + +markWithPotts <- function(kmin, gamma, nr, res, sq, subsize) { + ## Potts filtering on compact array; + ## kmin: minimal length of plateau + ## gamma: penalty for discontinuity + ## nr: number of values between breakpoints + ## res: sum of values between breakpoints + ## sq: sum of squares of values between breakpoints + + N <- length(nr) + Ant <- rep(0,N) + Sum <- rep(0,N) + Kvad <- rep(0,N) + Cost <- rep(0,N) + markSub <- rep(FALSE,N) + initAnt <- nr[1] + initSum <- res[1] + initKvad <- sq[1] + initAve <- initSum/initAnt + bestCost <- rep(0,N) + bestCost[1] <- initKvad - initSum*initAve + bestSplit <- rep(0,N) + k <- 2 + while(sum(nr[1:k]) < 2*kmin) { + Ant[2:k] <- Ant[2:k]+nr[k] + Sum[2:k]<-Sum[2:k]+res[k] + Kvad[2:k] <- Kvad[2:k]+sq[k] + bestCost[k] <- (initKvad+Kvad[2])-(initSum+Sum[2])^2/(initAnt+Ant[2]) + k <- k+1 + } + for (n in k:N) { + Ant[2:n] <- Ant[2:n]+nr[n] + Sum[2:n] <- Sum[2:n]+res[n] + Kvad[2:n] <- Kvad[2:n]+sq[n] + limit <- n + while(limit > 2 & Ant[limit] < kmin) {limit <- limit-1} + Cost[2:limit] <- bestCost[1:limit-1]+Kvad[2:limit]-Sum[2:limit]^2/Ant[2:limit] + Pos <- which.min(Cost[2:limit])+ 1 + cost <- Cost[Pos]+gamma + totCost <- (Kvad[2]+initKvad) - (Sum[2]+initSum)^2/(Ant[2]+initAnt) + if (totCost < cost) { + Pos <- 1 + cost <- totCost + } + bestCost[n] <- cost + bestSplit[n] <- Pos-1 + markSub[Pos-1] <- TRUE + } + help<-findMarks(markSub,nr,subsize) + return(help=help) +} + + +findMarks <- function(markSub,Nr,subsize){ + ## markSub: marks in compressed scale + ## NR: number of observations between potenstial breakpoints + mark<-rep(FALSE,subsize) ## marks in original scale + if(sum(markSub)<1) {return(mark)} else { + N<-length(markSub) + ant <- seq(1:N) + help <- ant[markSub] + lengdeHelp<-length(help) + help0 <- c(0,help[1:(lengdeHelp-1)]) + lengde <- help-help0 + start<-1 + oldStart<-1 + startOrig<-1 + for(i in 1:lengdeHelp){ + start <- start+lengde[i] + lengdeOrig <- sum(Nr[oldStart:(start-1)]) + startOrig <- startOrig+lengdeOrig + mark[startOrig-1]<-TRUE + oldStart<-start + } + return(mark) + } + +} + + +compact <- function(y,mark){ + ## accumulates numbers of observations, sums and + ## sums of squares between potential breakpoints + ## y: array to be compacted + ## mark: logical array of potential breakpoints + tell<-seq(1:length(y)) + cCTell<-tell[mark] + Ncomp<-length(cCTell) + lowTell<-c(0,cCTell[1:(Ncomp-1)]) + ant<-cCTell-lowTell + cy<-cumsum(y) + cCcy<-cy[mark] + lowcy<-c(0,cCcy[1:(Ncomp-1)]) + sum<-cCcy-lowcy + cy2<-cumsum(y^2) + cCcy2<-cy2[mark] + lowcy2<-c(0,cCcy2[1:(Ncomp-1)]) + sq<-cCcy2-lowcy2 + return(list(Nr=ant,Sum=sum,Sq=sq)) +} + +filterMarkS4 <- function(x,kmin,L,L2,frac1,frac2,frac3,thres){ + ## marks potential breakpoints, partially by a two 6*L and 6*L2 highpass + ## filters (L>L2), then by a filter seaching for potential kmin long segments + lengdeArr <- length(x) + xc<-cumsum(x) + xc<-c(0,xc) + ind11<-1:(lengdeArr-6*L+1) + ind12<-ind11+L + ind13<-ind11+3*L + ind14<-ind11+5*L + ind15<-ind11+6*L + + cost1<-abs(4*xc[ind13]-xc[ind11]-xc[ind12]-xc[ind14]-xc[ind15]) + cost1<-c(rep(0,3*L-1),cost1,rep(0,3*L)) + ##mark shortening in here + in1<-1:(lengdeArr-6) + in2<-in1+1 + in3<-in1+2 + in4<-in1+3 + in5<-in1+4 + in6<-in1+5 + in7<-in1+6 + test<-pmax(cost1[in1],cost1[in2],cost1[in3],cost1[in4],cost1[in5],cost1[in6],cost1[in7]) + test<-c(rep(0,3),test,rep(0,3)) + cost1B<-cost1[cost1>=thres*test] + frac1B<-min(0.8,frac1*length(cost1)/length(cost1B)) + limit <- quantile(cost1B,(1-frac1B),names=FALSE) + mark<-(cost1>limit)&(cost1>0.9*test) + + + ind21<-1:(lengdeArr-6*L2+1) + ind22<-ind21+L2 + ind23<-ind21+3*L2 + ind24<-ind21+5*L2 + ind25<-ind21+6*L2 + cost2<-abs(4*xc[ind23]-xc[ind21]-xc[ind22]-xc[ind24]-xc[ind25]) + limit2 <- quantile(cost2,(1-frac2),names=FALSE) + mark2<-(cost2>limit2) + mark2<-c(rep(0,3*L2-1),mark2,rep(0,3*L2)) + if(3*L>kmin){ + mark[kmin:(3*L-1)]<-TRUE + mark[(lengdeArr-3*L+1):(lengdeArr-kmin)]<-TRUE + } + else + { + mark[kmin]<- TRUE + mark[lengdeArr-kmin]<-TRUE + } + + if((kmin>1)&&(length(lengdeArr)>(3*kmin+1))){ + ind1<-1:(lengdeArr-3*kmin+1) + ind2<-ind1+3*kmin + ind3<-ind1+kmin + ind4<-ind1+2*kmin + shortAb <- abs(3*(xc[ind4]-xc[ind3])-(xc[ind2]-xc[ind1])) + in1<-1:(length(shortAb)-6) + in2<-in1+1 + in3<-in1+2 + in4<-in1+3 + in5<-in1+4 + in6<-in1+5 + in7<-in1+6 + test<-pmax(shortAb[in1],shortAb[in2],shortAb[in3],shortAb[in4],shortAb[in5],shortAb[in6],shortAb[in7]) + test<-c(rep(0,3),test,rep(0,3)) + cost1C<-shortAb[shortAb>=thres*test] + frac1C<-min(0.8,frac3*length(shortAb)/length(cost1C)) + limit3 <- quantile(cost1C,(1-frac1C),names=FALSE) + markH1<-(shortAb>limit3)&(shortAb>thres*test) + markH2<-c(rep(FALSE,(kmin-1)),markH1,rep(FALSE,2*kmin)) + markH3<-c(rep(FALSE,(2*kmin-1)),markH1,rep(FALSE,kmin)) + mark<-mark|mark2|markH2|markH3 + } else { + mark<-mark|mark2 + } + + if(3*L>kmin){ + mark[1:(kmin-1)]<-FALSE + mark[kmin:(3*L-1)]<-TRUE + mark[(lengdeArr-3*L+1):(lengdeArr-kmin)]<-TRUE + mark[(lengdeArr-kmin+1):(lengdeArr-1)]<-FALSE + mark[lengdeArr]<-TRUE + } + else + { + mark[1:(kmin-1)]<-FALSE + mark[(lengdeArr-kmin+1):(lengdeArr-1)]<-FALSE + mark[lengdeArr]<-TRUE + mark[kmin]<- TRUE + mark[lengdeArr-kmin]<-TRUE + } + + return(mark) +} + +medianFilter <- function(x,k){ + n <- length(x) + filtWidth <- 2*k + 1 + + #Make sure filtWidth does not exceed n + if(filtWidth > n){ + if(n==0){ + filtWidth <- 1 + }else if(n%%2 == 0){ + #runmed requires filtWidth to be odd, ensure this: + filtWidth <- n - 1 + }else{ + filtWidth <- n + } + } + + runMedian <- runmed(x,k=filtWidth,endrule="median") + + return(runMedian) +} + +getMad <- function(x,k=25){ + + #Remove observations that are equal to zero; are likely to be imputed, should not contribute to sd: + x <- x[x!=0] + + #Calculate runMedian + runMedian <- medianFilter(x,k) + + dif <- x-runMedian + SD <- mad(dif) + + return(SD) +} + +rearrangement.clustering_bedpe <- function(sv_bedpe, + kmin=10,# how many points at minimum in a peak, for the pcf algorithm + kmin.samples=kmin, # how many different samples at minimum in a peak + gamma.sdev=25, + PEAK.FACTOR=4, + thresh.dist=NA, + gamma=NA, + kmin.filter=kmin # if the pcf parameter is different from the definition of a peak +) { + + #add an id to the rearrangement + sv_bedpe$id <- 1:nrow(sv_bedpe) + + #functions below expect rows to be organised by chromosomes and ordered by position on the chromosome + + #prepare a dataframe for the calculation + rearrs.left <- sv_bedpe[,c('chrom1','start1','sample')] + names(rearrs.left ) <- NA + rearrs.right <- sv_bedpe[,c('chrom2','start2','sample')] + names(rearrs.right ) <- NA + rearrs.cncd <- rbind(rearrs.left , rearrs.right ) + colnames(rearrs.cncd) <- c('chr', 'position', 'sample') + rearrs.cncd$isLeft <- c(rep(TRUE, nrow(rearrs.left)), rep(FALSE, nrow(rearrs.left))) + rearrs.cncd$id <- c(sv_bedpe$id, sv_bedpe$id) + # sample.bps <- rearrs.cncd + #need to reorder + sample.bps <- NULL + for (chrom_i in unique(rearrs.cncd$chr)){ + tmptab <- rearrs.cncd[rearrs.cncd$chr==chrom_i,,drop=FALSE] + tmptab <- tmptab[order(tmptab$position),,drop=FALSE] + sample.bps <- rbind(sample.bps,tmptab) + } + rownames(sample.bps) <- 1:nrow(sample.bps) + + #run the algorithm + genome.size <- 3 * 10^9 + MIN.BPS <- 10 # minimal number of breakpoints on a chromosome to do any any segmentation + + logScale <- FALSE + + exp.dist <-genome.size/nrow(sample.bps) + + if (logScale) { + sample.bps$intermut.dist <- log10(calcIntermutDist(sample.bps, first.chrom.na=FALSE)$distPrev) # calculate the distances between the breakpoints + if (is.na(thresh.dist)) { + thresh.dist <- log10(exp.dist/PEAK.FACTOR) # calculate the threshold to call a peak + } + } else { + + sample.bps$intermut.dist <- calcIntermutDist(sample.bps, first.chrom.na=FALSE)$distPrev + if (is.na(thresh.dist)) { + thresh.dist <- exp.dist/PEAK.FACTOR + } + } + + + if (is.na(gamma) & !is.na(gamma.sdev)) { + # compute the mean absolute deviation + sdev <- getMad(sample.bps$intermut.dist); + gamma <- gamma.sdev*sdev + } + + + + sample.bps$is.clustered.single <- rep(FALSE, nrow(sample.bps)) + + all.kat.regions <- data.frame() + + for (chrom in unique(sample.bps$chr)) { # loop over chromosomes + + sample.bps.flag <- sample.bps$chr==chrom # breakpoints on a current chromosome + # sample.bps.chrom <- sample.bps[sample.bps.flag,] + # sample.bps.chrom <- sample.bps.chrom[order(sample.bps.chrom$position),] + # + if (sum(sample.bps.flag )>MIN.BPS ) { # if there are enough breakpoints on a chromosome to run pcf + + data.points <- sample.bps$intermut.dist[sample.bps.flag] + # data.points <- sample.bps.chrom$intermut.dist + + res = exactPcf(data.points, kmin, gamma, T) + + #reorder results + sample.bps$mean.intermut.dist[sample.bps.flag] <- res$yhat + + # prepare the points for pcf + subs <- data.frame(chr=sample.bps$chr[sample.bps.flag], pos=sample.bps$position[sample.bps.flag], sample=sample.bps$sample[sample.bps.flag]) + kat.regions <- extract.kat.regions(res, thresh.dist, subs, doMerging=TRUE, kmin.samples=1, kmin.filter= kmin.filter) # extract peaks, this is special case as we want at least kmin samples + + all.kat.regions <- rbind(all.kat.regions, kat.regions) + if (!is.null(kat.regions) && nrow( kat.regions )>0) { # if there are any kataegis regions found on this chormosome + for (k in 1:nrow(kat.regions)) { + + sample.bps$is.clustered.single[which(sample.bps.flag)[ kat.regions$firstBp[k] : kat.regions$lastBp[k]]] <- TRUE # call all breakpoints as clustered + } + } + } else { + + sample.bps$mean.intermut.dist[sample.bps.flag] <- mean(sample.bps$intermut.dist[sample.bps.flag]) + } + } + + + + if (!logScale) { # even if pcf was run on non-logged distances, I log the output + sample.bps$intermut.dist <- log10(sample.bps$intermut.dist) + sample.bps$mean.intermut.dist <- log10(sample.bps$mean.intermut.dist) + } + + # a rearrangement is in a cluster if any of its breakpoints are + sample.bps$is.clustered <- sample.bps$is.clustered.single + sample.bps$is.clustered[sample.bps$id %in% subset(sample.bps, is.clustered.single==TRUE)$id] <- TRUE + + # mark both breakpoints of a rearrangement as clustered if any is + sv_bedpe$is.clustered <- sv_bedpe$id %in% sample.bps$id[sample.bps$is.clustered] + + result <- list() + result$sv_bedpe <- sv_bedpe + result$kat.regions <- all.kat.regions + result +} + + +option_list <- list( + make_option(c("--kmin"), type="integer", default=10, help="How many points at minimum in a peak, for the pcf algorithm"), + make_option(c("--kmin_samples"), type="integer", default=1, help="How many different samples at minimum in a peak"), + make_option(c("--gamma_sdev"), type="integer", default=25, help="Gamma standard deviation"), + make_option(c("--peak_factor"), type="integer", default=10, help="Peak factor"), + make_option(c("--thresh_dist"), type="integer", default=NA, help="Threshold distance"), + make_option(c("--gamma"), type="integer", default=NA, help="Gamma"), + make_option(c("--kmin_filter"), type="integer", default=10, help="Kmin filter") +) +parser <- OptionParser(usage = "%prog [options] sv_bedpe out_fpath", option_list=option_list) +arguments <- parse_args(parser, positional_arguments = 2) +opt <- arguments$options +args <- arguments$args + +clustering(args[1], + args[2], + kmin=opt$kmin, + kmin.samples=opt$kmin_samples, + gamma.sdev=opt$gamma_sdev, + PEAK.FACTOR=opt$peak_factor, + thresh.dist=opt$thresh_dist, + gamma=opt$gamma, + kmin.filter=opt$kmin_filter) diff --git a/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile b/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile index f626cf00c63..c082c0dbc9a 100644 --- a/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile +++ b/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile @@ -13,7 +13,7 @@ WORKDIR /opt/opencga/signature.tools.lib RUN apt-get update -y && DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" apt-get install -y \ libcurl4 git libgmp-dev libcurl4-openssl-dev libgit2-dev libssl-dev libssh-dev libxml2-dev libfontconfig1-dev libharfbuzz-dev libfribidi-dev \ libfreetype6-dev libpng-dev libtiff5-dev libjpeg-dev \ - gnuplot pandoc samtools bcftools fastqc plink1.9 bwa r-base && \ + gnuplot pandoc samtools bcftools tabix fastqc plink1.9 bwa r-base && \ ## Installation dependencies using R install.packages() is slower than apt-get but final size is 400GB smaller. R -e "install.packages(c('BiocManager', 'RCircos', 'nnls', 'ggplot2', 'jsonlite', 'optparse', 'knitr', 'configr', 'dplyr', 'rmarkdown', 'tidyr', 'httr'))" && \ R -e "BiocManager::install('BiocStyle')" && \ @@ -21,10 +21,12 @@ RUN apt-get update -y && DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" apt R -e 'install.packages(c("devtools", "getopt"), repos="https://www.stats.bris.ac.uk/R/")' && \ git clone https://github.com/Nik-Zainal-Group/signature.tools.lib.git /opt/opencga/signature.tools.lib && \ git checkout d3d73db497b5b83abc55d6cd55840c34ed542628 && \ - sed -i '/BSgenome/d' DESCRIPTION && \ - R -e 'devtools::install(repos="https://www.stats.bris.ac.uk/R/")' && \ + sed -i '/Mmusculus/d' DESCRIPTION && \ + sed -i '/Cfamiliaris/d' DESCRIPTION && \ + sed -i '/1000genomes/d' DESCRIPTION && \ + R -e 'options(timeout = 300);devtools::install(repos="https://www.stats.bris.ac.uk/R/")' && \ ## Clean up rm -rf /var/lib/apt/lists/* /tmp/* /opt/opencga/signature.tools.lib/.git && \ strip --remove-section=.note.ABI-tag /usr/lib/x86_64-linux-gnu/libQt5Core.so.5 -WORKDIR /opt/opencga +WORKDIR /opt/opencga \ No newline at end of file diff --git a/opencga-app/pom.xml b/opencga-app/pom.xml index ccad8318be7..890b211ed9a 100644 --- a/opencga-app/pom.xml +++ b/opencga-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java index 5c743b66c9c..5554d04d94f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java @@ -36,7 +36,6 @@ import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.file.Paths; import java.util.Collections; @@ -178,7 +177,7 @@ private void install() throws CatalogException { try (CatalogManager catalogManager = new CatalogManager(configuration)) { catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), commandOptions.commonOptions.adminPassword, - commandOptions.email, commandOptions.organization, commandOptions.force, true); + commandOptions.email, commandOptions.organization, commandOptions.force); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java index a0562e7c9d3..f11d519c846 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java @@ -49,6 +49,7 @@ import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.GatkCommandOptions.GATK_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.GenomePlotCommandOptions.GENOME_PLOT_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.GwasCommandOptions.GWAS_RUN_COMMAND; +import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.HRDetectCommandOptions.HRDETECT_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.IndividualQcCommandOptions.INDIVIDUAL_QC_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.InferredSexCommandOptions.INFERRED_SEX_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.JulieRunCommandOptions.JULIE_RUN_COMMAND; @@ -169,6 +170,7 @@ public InternalCliOptionsParser() { variantSubCommands.addCommand(KNOCKOUT_RUN_COMMAND, variantCommandOptions.knockoutCommandOptions); variantSubCommands.addCommand(SAMPLE_ELIGIBILITY_RUN_COMMAND, variantCommandOptions.sampleEligibilityCommandOptions); variantSubCommands.addCommand(MUTATIONAL_SIGNATURE_RUN_COMMAND, variantCommandOptions.mutationalSignatureCommandOptions); + variantSubCommands.addCommand(HRDETECT_RUN_COMMAND, variantCommandOptions.hrDetectCommandOptions); variantSubCommands.addCommand(GENOME_PLOT_RUN_COMMAND, variantCommandOptions.genomePlotInternalCommandOptions); variantSubCommands.addCommand(MENDELIAN_ERROR_RUN_COMMAND, variantCommandOptions.mendelianErrorCommandOptions); variantSubCommands.addCommand(INFERRED_SEX_RUN_COMMAND, variantCommandOptions.inferredSexCommandOptions); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java index 1ed8c9fa275..3553384e71a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/AlignmentCommandExecutor.java @@ -156,8 +156,6 @@ private void qcRun() throws ToolException { ObjectMap params = new AlignmentQcParams( cliOptions.bamFile, - cliOptions.bedFile, - cliOptions.dictFile, cliOptions.skip, cliOptions.overwrite, cliOptions.outdir diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java index 552f0c60edd..8c6ac66073c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java @@ -34,6 +34,7 @@ import org.opencb.opencga.analysis.variant.VariantExportTool; import org.opencb.opencga.analysis.variant.genomePlot.GenomePlotAnalysis; import org.opencb.opencga.analysis.variant.gwas.GwasAnalysis; +import org.opencb.opencga.analysis.variant.hrdetect.HRDetectAnalysis; import org.opencb.opencga.analysis.variant.inferredSex.InferredSexAnalysis; import org.opencb.opencga.analysis.variant.julie.JulieTool; import org.opencb.opencga.analysis.variant.knockout.KnockoutAnalysis; @@ -89,6 +90,7 @@ import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.GatkCommandOptions.GATK_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.GenomePlotCommandOptions.GENOME_PLOT_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.GwasCommandOptions.GWAS_RUN_COMMAND; +import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.HRDetectCommandOptions.HRDETECT_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.IndividualQcCommandOptions.INDIVIDUAL_QC_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.InferredSexCommandOptions.INFERRED_SEX_RUN_COMMAND; import static org.opencb.opencga.app.cli.internal.options.VariantCommandOptions.KnockoutCommandOptions.KNOCKOUT_RUN_COMMAND; @@ -218,6 +220,9 @@ public void execute() throws Exception { case MUTATIONAL_SIGNATURE_RUN_COMMAND: mutationalSignature(); break; + case HRDETECT_RUN_COMMAND: + hrDetect(); + break; case GENOME_PLOT_RUN_COMMAND: genomePlot(); break; @@ -796,29 +801,52 @@ private void mutationalSignature() throws Exception { VariantCommandOptions.MutationalSignatureCommandOptions cliOptions = variantCommandOptions.mutationalSignatureCommandOptions; // Check signature release - checkSignatureVersion(cliOptions.sigVersion); + checkSignatureVersion(cliOptions.fitSigVersion); ObjectMap params = new MutationalSignatureAnalysisParams( cliOptions.id, cliOptions.description, + cliOptions.sample, cliOptions.query, - cliOptions.catalogues, - cliOptions.cataloguesContent, + cliOptions.fitId, cliOptions.fitMethod, - cliOptions.nBoot, - cliOptions.sigVersion, - cliOptions.organ, - cliOptions.thresholdPerc, - cliOptions.thresholdPval, - cliOptions.maxRareSigs, - cliOptions.signaturesFile, - cliOptions.rareSignaturesFile, + cliOptions.fitNBoot, + cliOptions.fitSigVersion, + cliOptions.fitOrgan, + cliOptions.fitThresholdPerc, + cliOptions.fitThresholdPval, + cliOptions.fitMaxRareSigs, + cliOptions.fitSignaturesFile, + cliOptions.fitRareSignaturesFile, + cliOptions.skip, cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); toolRunner.execute(MutationalSignatureAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); } + private void hrDetect() throws Exception { + VariantCommandOptions.HRDetectCommandOptions cliOptions = variantCommandOptions.hrDetectCommandOptions; + + ObjectMap params = new HRDetectAnalysisParams( + cliOptions.id, + cliOptions.description, + cliOptions.sample, + cliOptions.snvFittingId, + cliOptions.svFittingId, + cliOptions.cnvQuery, + cliOptions.indelQuery, + cliOptions.snv3CustomName, + cliOptions.snv8CustomName, + cliOptions.sv3CustomName, + cliOptions.sv8CustomName, + cliOptions.bootstrap, + cliOptions.outdir) + .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); + + toolRunner.execute(HRDetectAnalysis.class, params, Paths.get(cliOptions.outdir), jobId, token); + } + private void genomePlot() throws Exception { VariantCommandOptions.GenomePlotInternalCommandOptions cliOptions = variantCommandOptions.genomePlotInternalCommandOptions; @@ -911,7 +939,7 @@ private void sampleQc() throws Exception { VariantCommandOptions.SampleQcCommandOptions cliOptions = variantCommandOptions.sampleQcCommandOptions; // Check signature release - checkSignatureVersion(cliOptions.signatureSigVersion); + checkSignatureVersion(cliOptions.signatureFitSigVersion); // Build variant query from cli options AnnotationVariantQueryParams variantStatsQuery = ToolParams.fromParams(AnnotationVariantQueryParams.class, @@ -925,18 +953,20 @@ private void sampleQc() throws Exception { cliOptions.signatureId, cliOptions.signatureDescription, cliOptions.signatureQuery, + cliOptions.signatureFitId, cliOptions.signatureFitMethod, - cliOptions.signatureNBoot, - cliOptions.signatureSigVersion, - cliOptions.signatureOrgan, - cliOptions.signatureThresholdPerc, - cliOptions.signatureThresholdPval, - cliOptions.signatureMaxRareSigs, - cliOptions.signatureSignaturesFile, - cliOptions.signatureRareSignaturesFile, + cliOptions.signatureFitNBoot, + cliOptions.signatureFitSigVersion, + cliOptions.signatureFitOrgan, + cliOptions.signatureFitThresholdPerc, + cliOptions.signatureFitThresholdPval, + cliOptions.signatureFitMaxRareSigs, + cliOptions.signatureFitSignaturesFile, + cliOptions.signatureFitRareSignaturesFile, cliOptions.genomePlotId, cliOptions.genomePlotDescr, cliOptions.genomePlotConfigFile, + cliOptions.skip, cliOptions.outdir) .toObjectMap(cliOptions.commonOptions.params).append(ParamConstants.STUDY_PARAM, cliOptions.study); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java index a4c2251f6cb..cae48c1095e 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/VariantCommandOptions.java @@ -24,6 +24,7 @@ import org.opencb.opencga.analysis.variant.VariantExportTool; import org.opencb.opencga.analysis.variant.genomePlot.GenomePlotAnalysis; import org.opencb.opencga.analysis.variant.gwas.GwasAnalysis; +import org.opencb.opencga.analysis.variant.hrdetect.HRDetectAnalysis; import org.opencb.opencga.analysis.variant.inferredSex.InferredSexAnalysis; import org.opencb.opencga.analysis.variant.julie.JulieTool; import org.opencb.opencga.analysis.variant.knockout.KnockoutAnalysis; @@ -128,6 +129,7 @@ public class VariantCommandOptions { public final KnockoutCommandOptions knockoutCommandOptions; public final SampleEligibilityCommandOptions sampleEligibilityCommandOptions; public final MutationalSignatureCommandOptions mutationalSignatureCommandOptions; + public final HRDetectCommandOptions hrDetectCommandOptions; public final GenomePlotCommandOptions genomePlotCommandOptions; public final GenomePlotInternalCommandOptions genomePlotInternalCommandOptions; public final MendelianErrorCommandOptions mendelianErrorCommandOptions; @@ -196,6 +198,7 @@ public VariantCommandOptions(GeneralCliOptions.CommonCommandOptions commonComman this.knockoutCommandOptions = new KnockoutCommandOptions(); this.sampleEligibilityCommandOptions = new SampleEligibilityCommandOptions(); this.mutationalSignatureCommandOptions = new MutationalSignatureCommandOptions(); + this.hrDetectCommandOptions = new HRDetectCommandOptions(); this.genomePlotCommandOptions = new GenomePlotCommandOptions(); this.genomePlotInternalCommandOptions = new GenomePlotInternalCommandOptions(); this.mendelianErrorCommandOptions = new MendelianErrorCommandOptions(); @@ -1319,6 +1322,10 @@ public class MutationalSignatureCommandOptions { @Parameter(names = {"--study"}, description = "Study where all the samples belong to.") public String study; + @Parameter(names = {"--sample"}, description = "Sample ID.", required = true) + public String sample; + + // Signature catalogue @Parameter(names = {"--id"}, description = FieldConstants.MUTATIONAL_SIGNATURE_ID_DESCRIPTION) public String id; @@ -1329,40 +1336,96 @@ public class MutationalSignatureCommandOptions { public String query; // For fitting method - - @Parameter(names = {"--catalogues"}, description = FieldConstants.MUTATIONAL_SIGNATURE_CATALOGUES_DESCRIPTION) - public String catalogues; - - @Parameter(names = {"--catalogues-content"}, description = FieldConstants.MUTATIONAL_SIGNATURE_CATALOGUES_CONTENT_DESCRIPTION) - public String cataloguesContent; + @Parameter(names = {"--fit-id"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_ID_DESCRIPTION) + public String fitId; @Parameter(names = {"--fit-method"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_METHOD_DESCRIPTION) public String fitMethod = "FitMS"; - @Parameter(names = {"--n-boot"}, description = FieldConstants.MUTATIONAL_SIGNATURE_N_BOOT_DESCRIPTION) - public Integer nBoot; + @Parameter(names = {"--fit-n-boot"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_N_BOOT_DESCRIPTION) + public Integer fitNBoot; + + @Parameter(names = {"--fit-sig-version"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_SIG_VERSION_DESCRIPTION) + public String fitSigVersion = "RefSigv2"; + + @Parameter(names = {"--fit-organ"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_ORGAN_DESCRIPTION) + public String fitOrgan; + + @Parameter(names = {"--fit-threshold-perc"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_THRESHOLD_PERC_DESCRIPTION) + public Float fitThresholdPerc = 5f; + + @Parameter(names = {"--fit-threshold-pval"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_THRESHOLD_PVAL_DESCRIPTION) + public Float fitThresholdPval = 0.05f; + + @Parameter(names = {"--fit-max-rare-sigs"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_MAX_RARE_SIGS_DESCRIPTION) + public Integer fitMaxRareSigs = 1; + + @Parameter(names = {"--fit-signatures-file"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_SIGNATURES_FILE_DESCRIPTION) + public String fitSignaturesFile; + + @Parameter(names = {"--fit-rare-signatures-file"}, + description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_RARE_SIGNATURES_FILE_DESCRIPTION) + public String fitRareSignaturesFile; + + // Other + @Parameter(names = {"--skip"}, description = FieldConstants.SAMPLE_QUALITY_CONTROL_SKIP_DESCRIPTION) + public String skip; + + @Parameter(names = {"-o", "--outdir"}, description = FieldConstants.JOB_OUT_DIR_DESCRIPTION, arity = 1, required = false) + public String outdir; + } + + @Parameters(commandNames = HRDetectCommandOptions.HRDETECT_RUN_COMMAND, commandDescription = HRDetectAnalysis.DESCRIPTION) + public class HRDetectCommandOptions { + public static final String HRDETECT_RUN_COMMAND = HRDetectAnalysis.ID + "-run"; - @Parameter(names = {"--sig-version"}, description = FieldConstants.MUTATIONAL_SIGNATURE_SIG_VERSION_DESCRIPTION) - public String sigVersion = "RefSigv2"; + @ParametersDelegate + public GeneralCliOptions.CommonCommandOptions commonOptions = commonCommandOptions; + + @ParametersDelegate + public Object internalJobOptions = internalJobOptionsObject; + + @Parameter(names = {"--study"}, description = "Study where the sample belong to.") + public String study; + + @Parameter(names = {"--sample-id"}, description = FieldConstants.SAMPLE_ID_DESCRIPTION, required = true) + public String sample; - @Parameter(names = {"--organ"}, description = FieldConstants.MUTATIONAL_SIGNATURE_ORGAN_DESCRIPTION) - public String organ; + @Parameter(names = {"--id"}, description = FieldConstants.HRDETECT_ID_DESCRIPTION, arity = 1) + public String id; + + @Parameter(names = {"--description"}, description = FieldConstants.HRDETECT_DESCRIPTION_DESCRIPTION, arity = 1) + public String description; + + @Parameter(names = {"--snv-fitting-id"}, description = FieldConstants.HRDETECT_SNV_FITTING_ID_DESCRIPTION, required = true, + arity = 1) + public String snvFittingId; + + @Parameter(names = {"--sv-fitting-id"}, description = FieldConstants.HRDETECT_SV_FITTING_ID_DESCRIPTION, required = true, arity = 1) + public String svFittingId; - @Parameter(names = {"--threshold-perc"}, description = FieldConstants.MUTATIONAL_SIGNATURE_THRESHOLD_PERC_DESCRIPTION) - public Float thresholdPerc = 5f; + @Parameter(names = {"--cnv-query"}, description = FieldConstants.HRDETECT_CNV_QUERY_DESCRIPTION, required = true, arity = 1) + public String cnvQuery; - @Parameter(names = {"--threshold-pval"}, description = FieldConstants.MUTATIONAL_SIGNATURE_THRESHOLD_PVAL_DESCRIPTION) - public Float thresholdPval = 0.05f; + @Parameter(names = {"--indel-query"}, description = FieldConstants.HRDETECT_INDEL_QUERY_DESCRIPTION, required = true, arity = 1) + public String indelQuery; - @Parameter(names = {"--max-rare-sigs"}, description = FieldConstants.MUTATIONAL_SIGNATURE_MAX_RARE_SIGS_DESCRIPTION) - public Integer maxRareSigs = 1; + @Parameter(names = {"--snv3custom-name"}, description = FieldConstants.HRDETECT_SNV3_CUSTOM_NAME_DESCRIPTION, arity = 1) + public String snv3CustomName; - @Parameter(names = {"--signatures-file"}, description = FieldConstants.MUTATIONAL_SIGNATURE_SIGNATURES_FILE_DESCRIPTION) - public String signaturesFile; + @Parameter(names = {"--snv8custom-name"}, description = FieldConstants.HRDETECT_SNV8_CUSTOM_NAME_DESCRIPTION, arity = 1) + public String snv8CustomName; - @Parameter(names = {"--rare-signatures-file"}, description = FieldConstants.MUTATIONAL_SIGNATURE_RARE_SIGNATURES_FILE_DESCRIPTION) - public String rareSignaturesFile; + @Parameter(names = {"--sv3custom-name"}, description = FieldConstants.HRDETECT_SV3_CUSTOM_NAME_DESCRIPTION, arity = 1) + public String sv3CustomName; + @Parameter(names = {"--sv8custom-name"}, description = FieldConstants.HRDETECT_SV8_CUSTOM_NAME_DESCRIPTION, arity = 1) + public String sv8CustomName; + + @Parameter(names = {"--bootstrap"}, description = FieldConstants.HRDETECT_BOOTSTRAP_DESCRIPTION, arity = 1) + public Boolean bootstrap; + + // Other @Parameter(names = {"-o", "--outdir"}, description = FieldConstants.JOB_OUT_DIR_DESCRIPTION, arity = 1, required = false) public String outdir; } @@ -1595,32 +1658,36 @@ public class SampleQcCommandOptions { @Parameter(names = {"--ms-query"}, description = FieldConstants.MUTATIONAL_SIGNATURE_QUERY_DESCRIPTION) public String signatureQuery; + @Parameter(names = {"--ms-fit-id"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_ID_DESCRIPTION) + public String signatureFitId; + @Parameter(names = {"--ms-fit-method"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_METHOD_DESCRIPTION) public String signatureFitMethod = "FitMS"; - @Parameter(names = {"--ms-n-boot"}, description = FieldConstants.MUTATIONAL_SIGNATURE_N_BOOT_DESCRIPTION) - public Integer signatureNBoot; + @Parameter(names = {"--ms-fit-n-boot"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_N_BOOT_DESCRIPTION) + public Integer signatureFitNBoot; - @Parameter(names = {"--ms-sig-version"}, description = FieldConstants.MUTATIONAL_SIGNATURE_SIG_VERSION_DESCRIPTION) - public String signatureSigVersion = "RefSigv2"; + @Parameter(names = {"--ms-fit-sig-version"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_SIG_VERSION_DESCRIPTION) + public String signatureFitSigVersion = "RefSigv2"; - @Parameter(names = {"--ms-organ"}, description = FieldConstants.MUTATIONAL_SIGNATURE_ORGAN_DESCRIPTION) - public String signatureOrgan; + @Parameter(names = {"--ms-fit-organ"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_ORGAN_DESCRIPTION) + public String signatureFitOrgan; - @Parameter(names = {"--ms-threshold-perc"}, description = FieldConstants.MUTATIONAL_SIGNATURE_THRESHOLD_PERC_DESCRIPTION) - public Float signatureThresholdPerc = 5f; + @Parameter(names = {"--ms-fit-threshold-perc"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_THRESHOLD_PERC_DESCRIPTION) + public Float signatureFitThresholdPerc = 5f; - @Parameter(names = {"--ms-threshold-pval"}, description = FieldConstants.MUTATIONAL_SIGNATURE_THRESHOLD_PVAL_DESCRIPTION) - public Float signatureThresholdPval = 0.05f; + @Parameter(names = {"--ms-fit-threshold-pval"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_THRESHOLD_PVAL_DESCRIPTION) + public Float signatureFitThresholdPval = 0.05f; - @Parameter(names = {"--ms-max-rare-sigs"}, description = FieldConstants.MUTATIONAL_SIGNATURE_MAX_RARE_SIGS_DESCRIPTION) - public Integer signatureMaxRareSigs = 1; + @Parameter(names = {"--ms-fit-max-rare-sigs"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_MAX_RARE_SIGS_DESCRIPTION) + public Integer signatureFitMaxRareSigs = 1; - @Parameter(names = {"--ms-signatures-file"}, description = FieldConstants.MUTATIONAL_SIGNATURE_SIGNATURES_FILE_DESCRIPTION) - public String signatureSignaturesFile; + @Parameter(names = {"--ms-fit-signatures-file"}, description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_SIGNATURES_FILE_DESCRIPTION) + public String signatureFitSignaturesFile; - @Parameter(names = {"--ms-rare-signatures-file"}, description = FieldConstants.MUTATIONAL_SIGNATURE_RARE_SIGNATURES_FILE_DESCRIPTION) - public String signatureRareSignaturesFile; + @Parameter(names = {"--ms-fit-rare-signatures-file"}, + description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_RARE_SIGNATURES_FILE_DESCRIPTION) + public String signatureFitRareSignaturesFile; // Genome plot @@ -1633,6 +1700,10 @@ public class SampleQcCommandOptions { @Parameter(names = {"--gpcf", "--gp-config-file"}, description = FieldConstants.GENOME_PLOT_CONFIGURATION_FILE_DESCRIPTION) public String genomePlotConfigFile; + // Other + @Parameter(names = {"--skip"}, description = FieldConstants.SAMPLE_QUALITY_CONTROL_SKIP_DESCRIPTION) + public String skip; + @Parameter(names = {"-o", "--outdir"}, description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) public String outdir; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index 30ea9641968..8f6a7d65564 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022-12-13 OpenCB +* Copyright 2015-2023-01-10 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,7 @@ public abstract class OpenCgaCompleter implements Completer { .map(Candidate::new) .collect(toList()); - private List variantList = asList( "aggregationstats","annotation-metadata","annotation-query","circos-run","cohort-stats-delete","cohort-stats-info","cohort-stats-run","exomiser-run","export-run","family-genotypes","family-qc-run","file-delete","gatk-run","genome-plot-run","gwas-run","index-run","individual-qc-run","inferred-sex-run","knockout-gene-query","knockout-individual-query","knockout-run","mendelian-error-run","metadata","mutational-signature-query","mutational-signature-run","plink-run","query","relatedness-run","rvtests-run","sample-aggregation-stats","sample-eligibility-run","sample-qc-run","sample-query","sample-run","sample-stats-query","sample-stats-run","stats-export-run","stats-run") + private List variantList = asList( "aggregationstats","annotation-metadata","annotation-query","circos-run","cohort-stats-delete","cohort-stats-info","cohort-stats-run","exomiser-run","export-run","family-genotypes","family-qc-run","file-delete","gatk-run","genome-plot-run","gwas-run","hr-detect-run","index-run","individual-qc-run","inferred-sex-run","knockout-gene-query","knockout-individual-query","knockout-run","mendelian-error-run","metadata","mutational-signature-query","mutational-signature-run","plink-run","query","relatedness-run","rvtests-run","sample-aggregation-stats","sample-eligibility-run","sample-qc-run","sample-query","sample-run","sample-stats-query","sample-stats-run","stats-export-run","stats-run") .stream() .map(Candidate::new) .collect(toList()); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java index 6622d548c29..9b32de2528f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022-12-13 OpenCB +* Copyright 2015-2023-01-10 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -75,6 +75,7 @@ public OpencgaCliOptionsParser() { analysisVariantSubCommands.addCommand("gatk-run", analysisVariantCommandOptions.runGatkCommandOptions); analysisVariantSubCommands.addCommand("genome-plot-run", analysisVariantCommandOptions.runGenomePlotCommandOptions); analysisVariantSubCommands.addCommand("gwas-run", analysisVariantCommandOptions.runGwasCommandOptions); + analysisVariantSubCommands.addCommand("hr-detect-run", analysisVariantCommandOptions.runHrDetectCommandOptions); analysisVariantSubCommands.addCommand("index-run", analysisVariantCommandOptions.runIndexCommandOptions); analysisVariantSubCommands.addCommand("individual-qc-run", analysisVariantCommandOptions.runIndividualQcCommandOptions); analysisVariantSubCommands.addCommand("inferred-sex-run", analysisVariantCommandOptions.runInferredSexCommandOptions); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java index c25e53d68e9..3378e6f4e1f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java @@ -485,8 +485,6 @@ private RestResponse runQc() throws Exception { } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "bamFile",commandOptions.bamFile, true); - putNestedIfNotEmpty(beanParams, "bedFile",commandOptions.bedFile, true); - putNestedIfNotEmpty(beanParams, "dictFile",commandOptions.dictFile, true); putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); putNestedIfNotNull(beanParams, "overwrite",commandOptions.overwrite, true); putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java index 000ed437d45..b0238128d84 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java @@ -42,6 +42,7 @@ import org.opencb.opencga.core.models.variant.GatkWrapperParams; import org.opencb.opencga.core.models.variant.GenomePlotAnalysisParams; import org.opencb.opencga.core.models.variant.GwasAnalysisParams; +import org.opencb.opencga.core.models.variant.HRDetectAnalysisParams; import org.opencb.opencga.core.models.variant.IndividualQcAnalysisParams; import org.opencb.opencga.core.models.variant.InferredSexAnalysisParams; import org.opencb.opencga.core.models.variant.KnockoutAnalysisParams; @@ -137,6 +138,9 @@ public void execute() throws Exception { case "gwas-run": queryResponse = runGwas(); break; + case "hr-detect-run": + queryResponse = runHrDetect(); + break; case "index-run": queryResponse = runIndex(); break; @@ -783,6 +787,56 @@ private RestResponse runGwas() throws Exception { return openCGAClient.getVariantClient().runGwas(gwasAnalysisParams, queryParams); } + private RestResponse runHrDetect() throws Exception { + + logger.debug("Executing runHrDetect in Analysis - Variant command line"); + + AnalysisVariantCommandOptions.RunHrDetectCommandOptions commandOptions = analysisVariantCommandOptions.runHrDetectCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("jobId", commandOptions.jobId); + queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); + queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); + queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + HRDetectAnalysisParams hRDetectAnalysisParams= null; + if (commandOptions.jsonDataModel) { + hRDetectAnalysisParams = new HRDetectAnalysisParams(); + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(hRDetectAnalysisParams)); + return res; + } else if (commandOptions.jsonFile != null) { + hRDetectAnalysisParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), HRDetectAnalysisParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "sampleId",commandOptions.sampleId, true); + putNestedIfNotEmpty(beanParams, "snvFittingId",commandOptions.snvFittingId, true); + putNestedIfNotEmpty(beanParams, "svFittingId",commandOptions.svFittingId, true); + putNestedIfNotEmpty(beanParams, "cnvQuery",commandOptions.cnvQuery, true); + putNestedIfNotEmpty(beanParams, "indelQuery",commandOptions.indelQuery, true); + putNestedIfNotEmpty(beanParams, "snv3CustomName",commandOptions.snv3CustomName, true); + putNestedIfNotEmpty(beanParams, "snv8CustomName",commandOptions.snv8CustomName, true); + putNestedIfNotEmpty(beanParams, "sv3CustomName",commandOptions.sv3CustomName, true); + putNestedIfNotEmpty(beanParams, "sv8CustomName",commandOptions.sv8CustomName, true); + putNestedIfNotNull(beanParams, "bootstrap",commandOptions.bootstrap, true); + putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); + + hRDetectAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), HRDetectAnalysisParams.class); + } + return openCGAClient.getVariantClient().runHrDetect(hRDetectAnalysisParams, queryParams); + } + private RestResponse runIndex() throws Exception { logger.debug("Executing runIndex in Analysis - Variant command line"); @@ -1086,6 +1140,7 @@ private RestResponse queryMutationalSignature() throws Exception { ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("study", commandOptions.study); queryParams.putIfNotEmpty("sample", commandOptions.sample); + queryParams.putIfNotEmpty("type", commandOptions.type); queryParams.putIfNotEmpty("ct", commandOptions.ct); queryParams.putIfNotEmpty("biotype", commandOptions.biotype); queryParams.putIfNotEmpty("fileData", commandOptions.fileData); @@ -1099,17 +1154,8 @@ private RestResponse queryMutationalSignature() throws Exception { queryParams.putIfNotEmpty("panelFeatureType", commandOptions.panelFeatureType); queryParams.putIfNotEmpty("panelRoleInCancer", commandOptions.panelRoleInCancer); queryParams.putIfNotNull("panelIntersection", commandOptions.panelIntersection); - queryParams.putIfNotEmpty("catalogues", commandOptions.catalogues); - queryParams.putIfNotEmpty("cataloguesContent", commandOptions.cataloguesContent); - queryParams.putIfNotEmpty("fitMethod", commandOptions.fitMethod); - queryParams.putIfNotNull("nBoot", commandOptions.nBoot); - queryParams.putIfNotEmpty("sigVersion", commandOptions.sigVersion); - queryParams.putIfNotEmpty("organ", commandOptions.organ); - queryParams.putIfNotNull("thresholdPerc", commandOptions.thresholdPerc); - queryParams.putIfNotNull("thresholdPval", commandOptions.thresholdPval); - queryParams.putIfNotNull("maxRareSigs", commandOptions.maxRareSigs); - queryParams.putIfNotEmpty("signaturesFile", commandOptions.signaturesFile); - queryParams.putIfNotEmpty("rareSignaturesFile", commandOptions.rareSignaturesFile); + queryParams.putIfNotEmpty("msId", commandOptions.msId); + queryParams.putIfNotEmpty("msDescription", commandOptions.msDescription); if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } @@ -1148,18 +1194,19 @@ private RestResponse runMutationalSignature() throws Exception { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "id",commandOptions.id, true); putNestedIfNotEmpty(beanParams, "description",commandOptions.description, true); + putNestedIfNotEmpty(beanParams, "sample",commandOptions.sample, true); putNestedIfNotEmpty(beanParams, "query",commandOptions.query, true); - putNestedIfNotEmpty(beanParams, "catalogues",commandOptions.catalogues, true); - putNestedIfNotEmpty(beanParams, "cataloguesContent",commandOptions.cataloguesContent, true); + putNestedIfNotEmpty(beanParams, "fitId",commandOptions.fitId, true); putNestedIfNotEmpty(beanParams, "fitMethod",commandOptions.fitMethod, true); - putNestedIfNotNull(beanParams, "nBoot",commandOptions.nBoot, true); - putNestedIfNotEmpty(beanParams, "sigVersion",commandOptions.sigVersion, true); - putNestedIfNotEmpty(beanParams, "organ",commandOptions.organ, true); - putNestedIfNotNull(beanParams, "thresholdPerc",commandOptions.thresholdPerc, true); - putNestedIfNotNull(beanParams, "thresholdPval",commandOptions.thresholdPval, true); - putNestedIfNotNull(beanParams, "maxRareSigs",commandOptions.maxRareSigs, true); - putNestedIfNotEmpty(beanParams, "signaturesFile",commandOptions.signaturesFile, true); - putNestedIfNotEmpty(beanParams, "rareSignaturesFile",commandOptions.rareSignaturesFile, true); + putNestedIfNotNull(beanParams, "fitNBoot",commandOptions.fitNBoot, true); + putNestedIfNotEmpty(beanParams, "fitSigVersion",commandOptions.fitSigVersion, true); + putNestedIfNotEmpty(beanParams, "fitOrgan",commandOptions.fitOrgan, true); + putNestedIfNotNull(beanParams, "fitThresholdPerc",commandOptions.fitThresholdPerc, true); + putNestedIfNotNull(beanParams, "fitThresholdPval",commandOptions.fitThresholdPval, true); + putNestedIfNotNull(beanParams, "fitMaxRareSigs",commandOptions.fitMaxRareSigs, true); + putNestedIfNotEmpty(beanParams, "fitSignaturesFile",commandOptions.fitSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "fitRareSignaturesFile",commandOptions.fitRareSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); mutationalSignatureAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() @@ -1517,18 +1564,20 @@ private RestResponse runSampleQc() throws Exception { putNestedIfNotEmpty(beanParams, "msId",commandOptions.msId, true); putNestedIfNotEmpty(beanParams, "msDescription",commandOptions.msDescription, true); putNestedIfNotEmpty(beanParams, "msQuery",commandOptions.msQuery, true); + putNestedIfNotEmpty(beanParams, "msFitId",commandOptions.msFitId, true); putNestedIfNotEmpty(beanParams, "msFitMethod",commandOptions.msFitMethod, true); - putNestedIfNotNull(beanParams, "msNBoot",commandOptions.msNBoot, true); - putNestedIfNotEmpty(beanParams, "msSigVersion",commandOptions.msSigVersion, true); - putNestedIfNotEmpty(beanParams, "msOrgan",commandOptions.msOrgan, true); - putNestedIfNotNull(beanParams, "msThresholdPerc",commandOptions.msThresholdPerc, true); - putNestedIfNotNull(beanParams, "msThresholdPval",commandOptions.msThresholdPval, true); - putNestedIfNotNull(beanParams, "msMaxRareSigs",commandOptions.msMaxRareSigs, true); - putNestedIfNotEmpty(beanParams, "msSignaturesFile",commandOptions.msSignaturesFile, true); - putNestedIfNotEmpty(beanParams, "msRareSignaturesFile",commandOptions.msRareSignaturesFile, true); + putNestedIfNotNull(beanParams, "msFitNBoot",commandOptions.msFitNBoot, true); + putNestedIfNotEmpty(beanParams, "msFitSigVersion",commandOptions.msFitSigVersion, true); + putNestedIfNotEmpty(beanParams, "msFitOrgan",commandOptions.msFitOrgan, true); + putNestedIfNotNull(beanParams, "msFitThresholdPerc",commandOptions.msFitThresholdPerc, true); + putNestedIfNotNull(beanParams, "msFitThresholdPval",commandOptions.msFitThresholdPval, true); + putNestedIfNotNull(beanParams, "msFitMaxRareSigs",commandOptions.msFitMaxRareSigs, true); + putNestedIfNotEmpty(beanParams, "msFitSignaturesFile",commandOptions.msFitSignaturesFile, true); + putNestedIfNotEmpty(beanParams, "msFitRareSignaturesFile",commandOptions.msFitRareSignaturesFile, true); putNestedIfNotEmpty(beanParams, "gpId",commandOptions.gpId, true); putNestedIfNotEmpty(beanParams, "gpDescription",commandOptions.gpDescription, true); putNestedIfNotEmpty(beanParams, "gpConfigFile",commandOptions.gpConfigFile, true); + putNestedIfNotEmpty(beanParams, "skip",commandOptions.skip, true); putNestedIfNotEmpty(beanParams, "outdir",commandOptions.outdir, true); sampleQcAnalysisParams = JacksonUtils.getDefaultObjectMapper().copy() diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java index ad136880929..89a9d6aa6d4 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java @@ -29,7 +29,6 @@ import org.opencb.opencga.core.models.cohort.CohortCreateParams; import org.opencb.opencga.core.models.cohort.CohortGenerateParams; import org.opencb.opencga.core.models.cohort.CohortUpdateParams; -import org.opencb.opencga.core.models.common.Enums.CohortType; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; @@ -269,7 +268,7 @@ private RestResponse distinct() throws Exception { queryParams.putIfNotEmpty("id", commandOptions.id); queryParams.putIfNotEmpty("name", commandOptions.name); queryParams.putIfNotEmpty("uuid", commandOptions.uuid); - queryParams.putIfNotNull("type", commandOptions.type); + queryParams.putIfNotEmpty("type", commandOptions.type); queryParams.putIfNotEmpty("creationDate", commandOptions.creationDate); queryParams.putIfNotEmpty("modificationDate", commandOptions.modificationDate); queryParams.putIfNotNull("deleted", commandOptions.deleted); @@ -363,7 +362,7 @@ private RestResponse search() throws Exception { queryParams.putIfNotEmpty("id", commandOptions.id); queryParams.putIfNotEmpty("name", commandOptions.name); queryParams.putIfNotEmpty("uuid", commandOptions.uuid); - queryParams.putIfNotNull("type", commandOptions.type); + queryParams.putIfNotEmpty("type", commandOptions.type); queryParams.putIfNotEmpty("creationDate", commandOptions.creationDate); queryParams.putIfNotEmpty("modificationDate", commandOptions.modificationDate); queryParams.putIfNotNull("deleted", commandOptions.deleted); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java index 8847ad64f6c..74895f70573 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java @@ -18,7 +18,6 @@ import org.opencb.opencga.app.cli.main.options.OperationsVariantStorageCommandOptions; -import java.lang.Object; import org.opencb.biodata.models.variant.metadata.Aggregation; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.common.YesNoAuto; @@ -198,7 +197,7 @@ private RestResponse configureCellbase() throws Exception { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "url",commandOptions.url, true); putNestedIfNotEmpty(beanParams, "version",commandOptions.version, true); - putNestedIfNotEmpty(beanParams, "preferred",commandOptions.preferred, true); + putNestedIfNotEmpty(beanParams, "dataRelease",commandOptions.dataRelease, true); cellBaseConfiguration = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java index b13e71a5f34..4f2493a5ba8 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java @@ -18,7 +18,6 @@ import org.opencb.opencga.app.cli.main.options.ProjectsCommandOptions; -import java.lang.Object; import org.opencb.commons.datastore.core.FacetField; import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.models.project.Project; @@ -124,7 +123,7 @@ private RestResponse create() throws Exception { putNestedIfNotEmpty(beanParams, "organism.assembly",commandOptions.organismAssembly, true); putNestedIfNotEmpty(beanParams, "cellbase.url",commandOptions.cellbaseUrl, true); putNestedIfNotEmpty(beanParams, "cellbase.version",commandOptions.cellbaseVersion, true); - putNestedIfNotEmpty(beanParams, "cellbase.preferred",commandOptions.cellbasePreferred, true); + putNestedIfNotEmpty(beanParams, "cellbase.dataRelease",commandOptions.cellbaseDataRelease, true); putNestedIfNotNull(beanParams, "attributes",commandOptions.attributes, true); projectCreateParams = JacksonUtils.getDefaultObjectMapper().copy() diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java index 916f1b79a94..ac0c1b96c94 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisAlignmentCommandOptions.java @@ -433,7 +433,7 @@ public class RunPicardCommandOptions { } - @Parameters(commandNames = {"qc-run"}, commandDescription ="Compute quality control (QC) metrics for a given alignment file (including samtools stats, samtools flag stats, FastQC and HS metrics)") + @Parameters(commandNames = {"qc-run"}, commandDescription ="Compute quality control (QC) metrics for a given alignment file: samtools stats, samtools flag stats and FastQC metrics.") public class RunQcCommandOptions { @ParametersDelegate @@ -460,22 +460,16 @@ public class RunQcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--bam-file"}, description = "The body web service bamFile parameter", required = false, arity = 1) + @Parameter(names = {"--bam-file"}, description = "ID for the BAM file to process.", required = false, arity = 1) public String bamFile; - @Parameter(names = {"--bed-file"}, description = "The body web service bedFile parameter", required = false, arity = 1) - public String bedFile; - - @Parameter(names = {"--dict-file"}, description = "The body web service dictFile parameter", required = false, arity = 1) - public String dictFile; - - @Parameter(names = {"--skip"}, description = "The body web service skip parameter", required = false, arity = 1) + @Parameter(names = {"--skip"}, description = "To skip any alignment QC metrics use the following keywords (separated by commas): stats, flagstats, fastqc", required = false, arity = 1) public String skip; - @Parameter(names = {"--overwrite"}, description = "The body web service overwrite parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--overwrite"}, description = "To overwrite the QC metrics already computed.", required = false, help = true, arity = 0) public boolean overwrite = false; - @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index af334e224b1..c1abb8671f6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -48,6 +48,7 @@ public class AnalysisVariantCommandOptions { public RunGatkCommandOptions runGatkCommandOptions; public RunGenomePlotCommandOptions runGenomePlotCommandOptions; public RunGwasCommandOptions runGwasCommandOptions; + public RunHrDetectCommandOptions runHrDetectCommandOptions; public RunIndexCommandOptions runIndexCommandOptions; public RunIndividualQcCommandOptions runIndividualQcCommandOptions; public RunInferredSexCommandOptions runInferredSexCommandOptions; @@ -92,6 +93,7 @@ public AnalysisVariantCommandOptions(CommonCommandOptions commonCommandOptions, this.runGatkCommandOptions = new RunGatkCommandOptions(); this.runGenomePlotCommandOptions = new RunGenomePlotCommandOptions(); this.runGwasCommandOptions = new RunGwasCommandOptions(); + this.runHrDetectCommandOptions = new RunHrDetectCommandOptions(); this.runIndexCommandOptions = new RunIndexCommandOptions(); this.runIndividualQcCommandOptions = new RunIndividualQcCommandOptions(); this.runInferredSexCommandOptions = new RunInferredSexCommandOptions(); @@ -952,6 +954,74 @@ public class RunGwasCommandOptions { } + @Parameters(commandNames = {"hr-detect-run"}, commandDescription ="Run HRDetect analysis for a given somatic sample.") + public class RunHrDetectCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) + public String jobId; + + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) + public String jobDescription; + + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) + public String jobDependsOn; + + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) + public String jobTags; + + @Parameter(names = {"--id"}, description = "ID to identify the HRDetect results.", required = false, arity = 1) + public String id; + + @Parameter(names = {"--description"}, description = "Decription for these particular HRDetect results.", required = false, arity = 1) + public String description; + + @Parameter(names = {"--sample-id"}, description = "Sample data model hosts information about any biological material, normally extracted from an _Individual_, that is used for a particular analysis. This is the main data model, it stores the most basic and important information.", required = false, arity = 1) + public String sampleId; + + @Parameter(names = {"--snv-fitting-id"}, description = "Mutational signature fitting ID for SNV.", required = false, arity = 1) + public String snvFittingId; + + @Parameter(names = {"--sv-fitting-id"}, description = "Mutational signature fitting ID for SV.", required = false, arity = 1) + public String svFittingId; + + @Parameter(names = {"--cnv-query"}, description = "CNV query", required = false, arity = 1) + public String cnvQuery; + + @Parameter(names = {"--indel-query"}, description = "INDEL query", required = false, arity = 1) + public String indelQuery; + + @Parameter(names = {"--snv3custom-name"}, description = "Custom signature name that will be considered as SNV3 input for HRDetect.", required = false, arity = 1) + public String snv3CustomName; + + @Parameter(names = {"--snv8custom-name"}, description = "Custom signature name that will be considered as SNV8 input for HRDetect.", required = false, arity = 1) + public String snv8CustomName; + + @Parameter(names = {"--sv3custom-name"}, description = "Custom signature name that will be considered as SV3 input for HRDetect.", required = false, arity = 1) + public String sv3CustomName; + + @Parameter(names = {"--sv8custom-name"}, description = "Custom signature name that will be considered as SV8 input for HRDetect.", required = false, arity = 1) + public String sv8CustomName; + + @Parameter(names = {"--bootstrap"}, description = "Request HRDetect with bootstrap.", required = false, arity = 1) + public Boolean bootstrap; + + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) + public String outdir; + + } + @Parameters(commandNames = {"index-run"}, commandDescription =" [DEPRECATED] Use operation/variant/index") public class RunIndexCommandOptions { @@ -1334,6 +1404,9 @@ public class QueryMutationalSignatureCommandOptions { @Parameter(names = {"--sample"}, description = "Sample name", required = false, arity = 1) public String sample; + @Parameter(names = {"--type"}, description = "Variant type. Valid values: SNV, SV", required = false, arity = 1) + public String type; + @Parameter(names = {"--ct"}, description = "List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'", required = false, arity = 1) public String ct; @@ -1373,38 +1446,11 @@ public class QueryMutationalSignatureCommandOptions { @Parameter(names = {"--panel-intersection"}, description = "Intersect panel genes and regions with given genes and regions from que input query. This will prevent returning variants from regions out of the panel.", required = false, help = true, arity = 0) public boolean panelIntersection = false; - @Parameter(names = {"--catalogues"}, description = "File name containing mutational catalogues. Each sample catalogue is in a column, with sample names as column headers and channel.", required = false, arity = 1) - public String catalogues; - - @Parameter(names = {"--catalogues-content"}, description = "Mutational catalogues. Each sample catalogue is in a column, with sample names as column headers and channel.", required = false, arity = 1) - public String cataloguesContent; - - @Parameter(names = {"--fit-method"}, description = "Either Fit or FitMS. If not specified then FitMS", required = false, arity = 1) - public String fitMethod = "FitMS"; - - @Parameter(names = {"--n-boot"}, description = "Number of bootstrap to be used.", required = false, arity = 1) - public Integer nBoot; - - @Parameter(names = {"--sig-version"}, description = "Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2.", required = false, arity = 1) - public String sigVersion = "RefSigv2"; - - @Parameter(names = {"--organ"}, description = "When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus.", required = false, arity = 1) - public String organ; - - @Parameter(names = {"--threshold-perc"}, description = "Threshold in percentage of total mutations in a sample, only exposures larger than THRPERC are considered. If not specified 5.", required = false, arity = 1) - public Float thresholdPerc = 5f; - - @Parameter(names = {"--threshold-pval"}, description = "P-value to determine the empirical probability that the exposure is lower than the threshold. If not specified then 0.05.", required = false, arity = 1) - public Float thresholdPval = 0.05f; - - @Parameter(names = {"--max-rare-sigs"}, description = "Maximum number of rare signatures that are allowed to be present in each sample. If not specified 1.", required = false, arity = 1) - public Integer maxRareSigs = 1; - - @Parameter(names = {"--signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures.", required = false, arity = 1) - public String signaturesFile; + @Parameter(names = {"--ms-id"}, description = "Signature ID.", required = false, arity = 1) + public String msId; - @Parameter(names = {"--rare-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures.", required = false, arity = 1) - public String rareSignaturesFile; + @Parameter(names = {"--ms-description"}, description = "Signature description.", required = false, arity = 1) + public String msDescription; } @@ -1441,41 +1487,44 @@ public class RunMutationalSignatureCommandOptions { @Parameter(names = {"--description"}, description = "Signature description.", required = false, arity = 1) public String description; + @Parameter(names = {"--sample"}, description = "Sample.", required = false, arity = 1) + public String sample; + @Parameter(names = {"--query"}, description = "Signature query in JSON format, e.g: ''{\'sample\':\'NR123456_T\', \'fileData\': \'NR.123456_T_vs_NR.1234567_G.annot.vcf.gz:FILTER=PASS;CLPM<=0;ASMD>=140\'}'.", required = false, arity = 1) public String query; - @Parameter(names = {"--catalogues"}, description = "File name containing mutational catalogues. Each sample catalogue is in a column, with sample names as column headers and channel.", required = false, arity = 1) - public String catalogues; - - @Parameter(names = {"--catalogues-content"}, description = "Mutational catalogues. Each sample catalogue is in a column, with sample names as column headers and channel.", required = false, arity = 1) - public String cataloguesContent; + @Parameter(names = {"--fit-id"}, description = "Fiiting signature ID", required = false, arity = 1) + public String fitId; @Parameter(names = {"--fit-method"}, description = "Either Fit or FitMS. If not specified then FitMS", required = false, arity = 1) public String fitMethod = "FitMS"; - @Parameter(names = {"--n-boot"}, description = "Number of bootstrap to be used.", required = false, arity = 1) - public Integer nBoot; + @Parameter(names = {"--fit-n-boot"}, description = "Number of bootstrap to be used.", required = false, arity = 1) + public Integer fitNBoot; - @Parameter(names = {"--sig-version"}, description = "Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2.", required = false, arity = 1) - public String sigVersion = "RefSigv2"; + @Parameter(names = {"--fit-sig-version"}, description = "Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2.", required = false, arity = 1) + public String fitSigVersion = "RefSigv2"; - @Parameter(names = {"--organ"}, description = "When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus.", required = false, arity = 1) - public String organ; + @Parameter(names = {"--fit-organ"}, description = "When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus.", required = false, arity = 1) + public String fitOrgan; - @Parameter(names = {"--threshold-perc"}, description = "Threshold in percentage of total mutations in a sample, only exposures larger than THRPERC are considered. If not specified 5.", required = false, arity = 1) - public Float thresholdPerc = 5f; + @Parameter(names = {"--fit-threshold-perc"}, description = "Threshold in percentage of total mutations in a sample, only exposures larger than THRPERC are considered. If not specified 5.", required = false, arity = 1) + public Float fitThresholdPerc = 5f; - @Parameter(names = {"--threshold-pval"}, description = "P-value to determine the empirical probability that the exposure is lower than the threshold. If not specified then 0.05.", required = false, arity = 1) - public Float thresholdPval = 0.05f; + @Parameter(names = {"--fit-threshold-pval"}, description = "P-value to determine the empirical probability that the exposure is lower than the threshold. If not specified then 0.05.", required = false, arity = 1) + public Float fitThresholdPval = 0.05f; - @Parameter(names = {"--max-rare-sigs"}, description = "Maximum number of rare signatures that are allowed to be present in each sample. If not specified 1.", required = false, arity = 1) - public Integer maxRareSigs = 1; + @Parameter(names = {"--fit-max-rare-sigs"}, description = "Maximum number of rare signatures that are allowed to be present in each sample. If not specified 1.", required = false, arity = 1) + public Integer fitMaxRareSigs = 1; - @Parameter(names = {"--signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures.", required = false, arity = 1) - public String signaturesFile; + @Parameter(names = {"--fit-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures.", required = false, arity = 1) + public String fitSignaturesFile; - @Parameter(names = {"--rare-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures.", required = false, arity = 1) - public String rareSignaturesFile; + @Parameter(names = {"--fit-rare-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures.", required = false, arity = 1) + public String fitRareSignaturesFile; + + @Parameter(names = {"--skip"}, description = "To skip to compute catalogue counts or the signature fitting. Use the following keywords: catalogue, fitting.", required = false, arity = 1) + public String skip; @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; @@ -1977,7 +2026,7 @@ public class RunSampleQcCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--sample"}, description = "The body web service sample parameter", required = false, arity = 1) + @Parameter(names = {"--sample"}, description = "Sample data model hosts information about any biological material, normally extracted from an _Individual_, that is used for a particular analysis. This is the main data model, it stores the most basic and important information.", required = false, arity = 1) public String sample; @Parameter(names = {"--vs-id"}, description = "Variant stats ID.", required = false, arity = 1) @@ -2070,32 +2119,35 @@ public class RunSampleQcCommandOptions { @Parameter(names = {"--ms-query"}, description = "Signature query in JSON format, e.g: ''{\'sample\':\'NR123456_T\', \'fileData\': \'NR.123456_T_vs_NR.1234567_G.annot.vcf.gz:FILTER=PASS;CLPM<=0;ASMD>=140\'}'.", required = false, arity = 1) public String msQuery; + @Parameter(names = {"--ms-fit-id"}, description = "Either Fit or FitMS. If not specified then FitMS", required = false, arity = 1) + public String msFitId; + @Parameter(names = {"--ms-fit-method"}, description = "Either Fit or FitMS. If not specified then FitMS", required = false, arity = 1) public String msFitMethod = "FitMS"; - @Parameter(names = {"--ms-n-boot"}, description = "Number of bootstrap to be used.", required = false, arity = 1) - public Integer msNBoot; + @Parameter(names = {"--ms-fit-n-boot"}, description = "Number of bootstrap to be used.", required = false, arity = 1) + public Integer msFitNBoot; - @Parameter(names = {"--ms-sig-version"}, description = "Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2.", required = false, arity = 1) - public String msSigVersion = "RefSigv2"; + @Parameter(names = {"--ms-fit-sig-version"}, description = "Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2.", required = false, arity = 1) + public String msFitSigVersion = "RefSigv2"; - @Parameter(names = {"--ms-organ"}, description = "When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus.", required = false, arity = 1) - public String msOrgan; + @Parameter(names = {"--ms-fit-organ"}, description = "When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus.", required = false, arity = 1) + public String msFitOrgan; - @Parameter(names = {"--ms-threshold-perc"}, description = "Threshold in percentage of total mutations in a sample, only exposures larger than THRPERC are considered. If not specified 5.", required = false, arity = 1) - public Float msThresholdPerc = 5f; + @Parameter(names = {"--ms-fit-threshold-perc"}, description = "Threshold in percentage of total mutations in a sample, only exposures larger than THRPERC are considered. If not specified 5.", required = false, arity = 1) + public Float msFitThresholdPerc = 5f; - @Parameter(names = {"--ms-threshold-pval"}, description = "P-value to determine the empirical probability that the exposure is lower than the threshold. If not specified then 0.05.", required = false, arity = 1) - public Float msThresholdPval = 0.05f; + @Parameter(names = {"--ms-fit-threshold-pval"}, description = "P-value to determine the empirical probability that the exposure is lower than the threshold. If not specified then 0.05.", required = false, arity = 1) + public Float msFitThresholdPval = 0.05f; - @Parameter(names = {"--ms-max-rare-sigs"}, description = "Maximum number of rare signatures that are allowed to be present in each sample. If not specified 1.", required = false, arity = 1) - public Integer msMaxRareSigs = 1; + @Parameter(names = {"--ms-fit-max-rare-sigs"}, description = "Maximum number of rare signatures that are allowed to be present in each sample. If not specified 1.", required = false, arity = 1) + public Integer msFitMaxRareSigs = 1; - @Parameter(names = {"--ms-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures.", required = false, arity = 1) - public String msSignaturesFile; + @Parameter(names = {"--ms-fit-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures.", required = false, arity = 1) + public String msFitSignaturesFile; - @Parameter(names = {"--ms-rare-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures.", required = false, arity = 1) - public String msRareSignaturesFile; + @Parameter(names = {"--ms-fit-rare-signatures-file"}, description = "The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures.", required = false, arity = 1) + public String msFitRareSignaturesFile; @Parameter(names = {"--gp-id"}, description = "Genome plot ID.", required = false, arity = 1) public String gpId; @@ -2106,6 +2158,9 @@ public class RunSampleQcCommandOptions { @Parameter(names = {"--gp-config-file"}, description = "Genome plot configuration file.", required = false, arity = 1) public String gpConfigFile; + @Parameter(names = {"--skip"}, description = "Quality control metrics to skip. Valid values are: variant-stats, signature, signature-catalogue, signature-fitting, genome-plot", required = false, arity = 1) + public String skip; + @Parameter(names = {"--outdir"}, description = "Output dir for the job.", required = false, arity = 1) public String outdir; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java index 2c7d26f79aa..4d9591c830a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java @@ -119,14 +119,14 @@ public class ConfigureCellbaseCommandOptions { @Parameter(names = {"--annotation-save-id"}, description = "Save a copy of the current variant annotation at the database", required = false, arity = 1) public String annotationSaveId; - @Parameter(names = {"--url"}, description = "The body web service url parameter", required = false, arity = 1) + @Parameter(names = {"--url"}, description = "URL to CellBase REST web services, by default official ZettaGenomics installation is used", required = false, arity = 1) public String url; - @Parameter(names = {"--version"}, description = "The body web service version parameter", required = false, arity = 1) + @Parameter(names = {"--version"}, description = "URL to CellBase REST web services, by default official ZettaGenomics installation is used", required = false, arity = 1) public String version; - @Parameter(names = {"--preferred"}, description = "The body web service preferred parameter", required = false, arity = 1) - public String preferred; + @Parameter(names = {"--data-release"}, description = "CellBase data release version to be used. If empty, will use the active one", required = false, arity = 1) + public String dataRelease; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ProjectsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ProjectsCommandOptions.java index 1d07185da93..f3ae96c6401 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ProjectsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ProjectsCommandOptions.java @@ -101,14 +101,14 @@ public class CreateCommandOptions { @Parameter(names = {"--organism-assembly"}, description = "The body web service assembly parameter", required = false, arity = 1) public String organismAssembly; - @Parameter(names = {"--cellbase-url"}, description = "The body web service url parameter", required = false, arity = 1) + @Parameter(names = {"--cellbase-url"}, description = "URL to CellBase REST web services, by default official ZettaGenomics installation is used", required = false, arity = 1) public String cellbaseUrl; - @Parameter(names = {"--cellbase-version"}, description = "The body web service version parameter", required = false, arity = 1) + @Parameter(names = {"--cellbase-version"}, description = "URL to CellBase REST web services, by default official ZettaGenomics installation is used", required = false, arity = 1) public String cellbaseVersion; - @Parameter(names = {"--cellbase-preferred"}, description = "The body web service preferred parameter", required = false, arity = 1) - public String cellbasePreferred; + @Parameter(names = {"--cellbase-data-release"}, description = "CellBase data release version to be used. If empty, will use the active one", required = false, arity = 1) + public String cellbaseDataRelease; @DynamicParameter(names = {"--attributes"}, description = "The body web service attributes parameter. Use: --attributes key=value", required = false) public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_11/catalog/SignatureFittingsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_11/catalog/SignatureFittingsMigration.java new file mode 100644 index 00000000000..46fcf8aa891 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_4_11/catalog/SignatureFittingsMigration.java @@ -0,0 +1,61 @@ +package org.opencb.opencga.app.migrations.v2_4_11.catalog; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.apache.commons.collections4.CollectionUtils; +import org.bson.Document; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static com.mongodb.client.model.Filters.eq; + +@Migration(id = "signature_fittings" , + description = "Replace fitting for fittings in Signature", + version = "2.4.11", + domain = Migration.MigrationDomain.CATALOG, + language = Migration.MigrationLanguage.JAVA, + date = 20221109 +) +public class SignatureFittingsMigration extends MigrationTool { + @Override + protected void run() throws Exception { + migrateCollection(Arrays.asList(MongoDBAdaptorFactory.SAMPLE_COLLECTION, MongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, + MongoDBAdaptorFactory.DELETED_SAMPLE_COLLECTION), + Filters.exists("qualityControl.variant.signatures.fitting"), + Projections.include(Collections.singletonList("qualityControl.variant.signatures")), + (document, bulk) -> { + Document qc = document.get("qualityControl", Document.class); + if (qc == null) { + return; + } + Document variant = qc.get("variant", Document.class); + if (variant == null) { + return; + } + List signatures = variant.getList("signatures", Document.class); + if (CollectionUtils.isNotEmpty(signatures)) { + for (Document signature : signatures) { + if (signature != null) { + Document fitting = signature.get("fitting", Document.class); + if (fitting != null) { + fitting.put("id", "fitting-1"); + signature.put("fittings", Collections.singletonList(fitting)); + signature.remove("fitting"); + } + } + } + + bulk.add(new UpdateOneModel<>( + eq("_id", document.get("_id")), + new Document("$set", new Document("qualityControl.variant", signatures))) + ); + } + }); + } +} diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_6_0/storage/AddCellbaseDataRelease.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_6_0/storage/AddCellbaseDataRelease.java new file mode 100644 index 00000000000..8c949dd7214 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_6_0/storage/AddCellbaseDataRelease.java @@ -0,0 +1,54 @@ +package org.opencb.opencga.app.migrations.v2_6_0.storage; + +import org.apache.solr.common.StringUtils; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.core.config.storage.CellBaseConfiguration; +import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.storage.core.utils.CellBaseUtils; + +@Migration(id = "add_cellbase_data_release" , + description = "Add default cellbase data release if missing", + version = "2.6.0", + domain = Migration.MigrationDomain.STORAGE, + language = Migration.MigrationLanguage.JAVA, + date = 20230104 +) +public class AddCellbaseDataRelease extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + VariantStorageManager variantStorageManager = getVariantStorageManager(); + for (String projectFqn : getVariantStorageProjects()) { + Project project = catalogManager.getProjectManager().get(projectFqn, new QueryOptions(), token).first(); + CellBaseConfiguration cellbase = project.getCellbase(); + CellBaseUtils cellBaseUtils = getVariantStorageEngineByProject(projectFqn).getCellBaseUtils(); + boolean updateCellbase = false; + if (cellbase == null) { + cellbase = new CellBaseConfiguration(cellBaseUtils.getURL(), cellBaseUtils.getVersion()); + updateCellbase = true; + } + + if (StringUtils.isEmpty(cellbase.getDataRelease())) { + if (cellBaseUtils.getDataRelease() != null) { + cellbase.setDataRelease(cellBaseUtils.getDataRelease()); + updateCellbase = true; + } else { + if (cellBaseUtils.supportsDataRelease()) { + cellbase.setDataRelease("1"); + updateCellbase = true; + } else { + String serverVersion = cellBaseUtils.getVersionFromServer(); + logger.info("DataRelease not supported on version '" + serverVersion + "' . Leaving empty"); + } + } + } + if (updateCellbase) { + logger.info("Update cellbase info for project '{}' with '{}'", projectFqn, cellBaseUtils); + variantStorageManager.setCellbaseConfiguration(projectFqn, null, false, null, token); + } + } + } +} diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 95349542f1e..8e02abe7ffa 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java index 6169de00cb9..86ccefa2653 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java @@ -20,6 +20,7 @@ import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.config.Admin; import org.opencb.opencga.core.config.Configuration; import java.util.Map; @@ -36,20 +37,27 @@ public interface DBAdaptorFactory extends AutoCloseable { boolean isCatalogDBReady(); /** - * Installs the catalog database with their corresponding indexes. + * Create all collections for the database. * * @param configuration Configuration of catalog. * @throws CatalogException if there is any problem with the installation. */ - void installCatalogDB(Configuration configuration) throws CatalogException; + void createAllCollections(Configuration configuration) throws CatalogException; + + /** + * Initialise meta collection. + * + * @param admin Admin information. + * @throws CatalogException if there is any problem with the installation. + */ + void initialiseMetaCollection(Admin admin) throws CatalogException; /** * Creates the indexes needed to make queries faster. * - * @param wholeIndexes boolean indicating whether to generate all indexes or only the unique ones. * @throws CatalogDBException if there is any problem creating the indexes. */ - void createIndexes(boolean wholeIndexes) throws CatalogDBException; + void createIndexes() throws CatalogDBException; String getCatalogDatabase(String prefix); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuditMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuditMongoDBAdaptor.java index 1000250a013..0f462088c5f 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuditMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuditMongoDBAdaptor.java @@ -16,18 +16,17 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.model.Filters; import org.bson.Document; import org.bson.conversions.Bson; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.mongodb.MongoDBCollection; -import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.catalog.db.api.AuditDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.OpenCgaMongoConverter; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.LoggerFactory; @@ -83,7 +82,7 @@ public OpenCGAResult insertAuditRecords(List auditReco @Override public OpenCGAResult get(Query query, QueryOptions queryOptions) throws CatalogDBException { Bson bson = parseQuery(query); - logger.debug("Audit query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Audit query: {}", bson.toBsonDocument()); return new OpenCGAResult<>(auditCollection.find(bson, auditConverter, queryOptions)); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java index 2bc40b93616..edab86e2f36 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; @@ -207,7 +206,7 @@ private EntryPermission internalGet(long resourceId, List membersList, E } for (Bson bson : aggregation) { - logger.debug("Get Acl: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Get Acl: {}", bson.toBsonDocument()); } DataResult aggregate = collection.aggregate(aggregation, null); @@ -378,8 +377,7 @@ public OpenCGAResult removeFromStudy(long studyId, String member, Enums.Resou .append(QueryParams.USER_DEFINED_ACLS.key(), removePermissions) ); logger.debug("Remove all acls for entity {} for member {} in study {}. Query: {}, pullAll: {}", resource, member, studyId, - query.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + query.toBsonDocument(), update.toBsonDocument()); return new OpenCGAResult<>(update(null, query, update, resource)); } @@ -436,9 +434,7 @@ private void setToMembers(List resourceIds, List members, List resourceIds, List members, List resourceIds, List members, List resourceI update = new Document("$pullAll", new Document(QueryParams.ACL.key(), removePermissions)); } - logger.debug("Remove Acls (pullAll): Query {}, Pull {}", - queryDocument.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Remove Acls (pullAll): Query {}, Pull {}", queryDocument.toBsonDocument(), update.toBsonDocument()); update(clientSession, queryDocument, update, resource); } @@ -635,9 +626,7 @@ public OpenCGAResult setAcls(List resourceIds, AclEntryList acls, Enums update = new Document("$set", new Document(QueryParams.ACL.key(), permissionArray)); } - logger.debug("Set Acls (set): Query {}, Push {}", - queryDocument.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Set Acls (set): Query {}, Push {}", queryDocument.toBsonDocument(), update.toBsonDocument()); update(null, queryDocument, update, resource); } @@ -676,9 +665,7 @@ public OpenCGAResult applyPermissionRules(long studyId, PermissionRule permis .append(QueryParams.ACL.key(), new Document("$each", myPermissions)) .append(PERMISSION_RULES_APPLIED, permissionRule.getId())); - logger.debug("Apply permission rules: Query {}, Update {}", - bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Apply permission rules: Query {}, Update {}", bson.toBsonDocument(), update.toBsonDocument()); return new OpenCGAResult<>(update(null, bson, update, entry.getResource())); } @@ -754,9 +741,8 @@ public OpenCGAResult removePermissionRuleAndRemovePermissions(Study study, Strin .append(QueryParams.USER_DEFINED_ACLS.key(), manualPermissions) .append(PERMISSION_RULES_APPLIED, permissionRulesApplied)); - logger.debug("Remove permission rule id and permissions from {}: Query {}, Update {}", entry, - tmpQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Remove permission rule id and permissions from {}: Query {}, Update {}", entry, tmpQuery.toBsonDocument(), + update.toBsonDocument()); DataResult result = update(null, tmpQuery, update, entry.getResource()); if (result.getNumUpdated() == 0) { throw new CatalogException("Could not update and remove permission rule from entry " + myDocument.get(PRIVATE_UID)); @@ -837,8 +823,7 @@ public OpenCGAResult removePermissionRuleAndRestorePermissions(Study study, Stri .append(PERMISSION_RULES_APPLIED, permissionRulesApplied)); logger.debug("Remove permission rule id and restoring permissions from {}: Query {}, Update {}", entry, - tmpQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + tmpQuery.toBsonDocument(), update.toBsonDocument()); DataResult result = update(null, tmpQuery, update, entry.getResource()); if (result.getNumUpdated() == 0) { throw new CatalogException("Could not update and remove permission rule from entry " + myDocument.get(PRIVATE_UID)); @@ -862,9 +847,8 @@ public OpenCGAResult removePermissionRule(long studyId, String permissionRuleToD .append(PERMISSION_RULES_APPLIED, permissionRuleId); Document update = new Document() .append("$pull", new Document(PERMISSION_RULES_APPLIED, permissionRuleId)); - logger.debug("Remove permission rule id from all {} in study {}: Query {}, Update {}", entry, studyId, - query.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Remove permission rule id from all {} in study {}: Query {}, Update {}", entry, studyId, query.toBsonDocument(), + update.toBsonDocument()); DataResult result = update(null, query, update, entry.getResource()); if (result.getNumUpdated() == 0) { @@ -895,9 +879,8 @@ private void removeReferenceToPermissionRuleInStudy(long studyId, String permiss Document update = new Document("$pull", new Document(StudyDBAdaptor.QueryParams.PERMISSION_RULES.key() + "." + entry, new Document("id", permissionRuleToDelete))); - logger.debug("Remove permission rule from the study {}: Query {}, Update {}", studyId, - query.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Remove permission rule from the study {}: Query {}, Update {}", studyId, query.toBsonDocument(), + update.toBsonDocument()); DataResult result = update(null, query, update, Enums.Resource.STUDY); if (result.getNumUpdated() == 0) { throw new CatalogException("Could not remove permission rule " + permissionRuleToDelete + " from study " + studyId); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index 778bacfdce2..1318e79df82 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; @@ -169,7 +168,7 @@ public OpenCGAResult count(final Query query, final String user) OpenCGAResult count(ClientSession clientSession, final Query query, final String user) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Bson bson = parseQuery(query, user); - logger.debug("Clinical count: query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Clinical count: query : {}", bson.toBsonDocument()); return new OpenCGAResult<>(clinicalCollection.count(clientSession, bson)); } @@ -236,8 +235,7 @@ OpenCGAResult update(ClientSession clientSession, ClinicalAnalysis clinical, Obj if (!updateOperation.isEmpty()) { Bson bsonQuery = Filters.eq(PRIVATE_UID, clinicalAnalysisUid); - logger.debug("Update clinical analysis. Query: {}, Update: {}", bsonQuery.toBsonDocument(Document.class, - MongoClient.getDefaultCodecRegistry()), updateDocument); + logger.debug("Update clinical analysis. Query: {}, Update: {}", bsonQuery.toBsonDocument(), updateDocument); update = clinicalCollection.update(clientSession, bsonQuery, updateOperation, null); if (update.getNumMatches() == 0) { @@ -262,7 +260,7 @@ OpenCGAResult update(ClientSession clientSession, ClinicalAnalysis clinical, Obj Bson bsonQuery = parseQuery(nestedDocument.getQuery().append(QueryParams.UID.key(), clinicalAnalysisUid)); logger.debug("Update nested element from Clinical Analysis. Query: {}, Update: {}", - bsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), nestedDocument.getSet()); + bsonQuery.toBsonDocument(), nestedDocument.getSet()); update = clinicalCollection.update(clientSession, bsonQuery, nestedDocument.getSet(), null); @@ -745,7 +743,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu qOptions = removeInnerProjections(qOptions, QueryParams.INTERPRETATION.key()); qOptions = removeInnerProjections(qOptions, QueryParams.SECONDARY_INTERPRETATIONS.key()); - logger.debug("Clinical analysis query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Clinical analysis query : {}", bson.toBsonDocument()); if (!query.getBoolean(QueryParams.DELETED.key())) { return clinicalCollection.iterator(clientSession, bson, null, null, qOptions); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java index cc8a974fa4c..63ff15c96fb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import org.apache.commons.collections4.CollectionUtils; @@ -212,7 +211,7 @@ public OpenCGAResult count(final Query query, final String user) private OpenCGAResult count(ClientSession clientSession, final Query query, final String user) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Bson bson = parseQuery(query, user); - logger.debug("Cohort count: query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Cohort count: query : {}", bson.toBsonDocument()); return new OpenCGAResult<>(cohortCollection.count(clientSession, bson)); } @@ -308,9 +307,7 @@ private OpenCGAResult privateUpdate(ClientSession clientSession, Cohort List events = new ArrayList<>(); if (!cohortUpdate.isEmpty()) { Bson finalQuery = parseQuery(tmpQuery); - logger.debug("Cohort update: query : {}, update: {}", - finalQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - cohortUpdate.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Cohort update: query : {}, update: {}", finalQuery.toBsonDocument(), cohortUpdate.toBsonDocument()); result = cohortCollection.update(clientSession, finalQuery, cohortUpdate, null); if (result.getNumMatches() == 0) { @@ -762,7 +759,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu qOptions = filterOptions(qOptions, FILTER_ROUTE_COHORTS); fixAclProjection(qOptions); - logger.debug("Cohort query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Cohort query : {}", bson.toBsonDocument()); if (!query.getBoolean(QueryParams.DELETED.key())) { return cohortCollection.iterator(clientSession, bson, null, null, qOptions); } else { @@ -985,9 +982,7 @@ void removeSampleReferences(ClientSession clientSession, long studyUid, long sam QueryOptions multi = new QueryOptions(MongoDBCollection.MULTI, true); - logger.debug("Sample references extraction. Query: {}, update: {}", - bsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Sample references extraction. Query: {}, update: {}", bsonQuery.toBsonDocument(), update.toBsonDocument()); DataResult result = cohortCollection.update(clientSession, bsonQuery, update, multi); logger.debug("Sample uid '" + sampleUid + "' references removed from " + result.getNumUpdated() + " out of " + result.getNumMatches() + " cohorts"); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java index f9f612dbac0..a9958881493 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; @@ -251,7 +250,7 @@ public OpenCGAResult count(final Query query, final String user) public OpenCGAResult count(ClientSession clientSession, final Query query, final String user) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Bson bson = parseQuery(query, user); - logger.debug("Family count: query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Family count: query : {}", bson.toBsonDocument()); return new OpenCGAResult<>(familyCollection.count(clientSession, bson)); } @@ -421,9 +420,7 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Family family, if (!familyUpdate.isEmpty()) { Bson finalQuery = parseQuery(tmpQuery); - logger.debug("Family update: query : {}, update: {}", - finalQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - familyUpdate.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Family update: query : {}, update: {}", finalQuery.toBsonDocument(), familyUpdate.toBsonDocument()); result = familyCollection.update(clientSession, finalQuery, familyUpdate, new QueryOptions("multi", true)); if (parameters.containsKey(QueryParams.ID.key())) { @@ -921,7 +918,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu qOptions = removeAnnotationProjectionOptions(qOptions); fixAclProjection(qOptions); - logger.debug("Family query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Family query : {}", bson.toBsonDocument()); MongoDBCollection collection = getQueryCollection(query, familyCollection, archiveFamilyCollection, deletedFamilyCollection); return collection.iterator(clientSession, bson, null, null, qOptions); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index 99ac887b952..76719b7dc1d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Updates; @@ -361,9 +360,7 @@ OpenCGAResult privateUpdate(ClientSession clientSession, File file, Obje List events = new ArrayList<>(); if (!fileUpdate.isEmpty()) { - logger.debug("Update file. Query: {}, Update: {}", - queryBson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - fileUpdate.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Update file. Query: {}, Update: {}", queryBson.toBsonDocument(), fileUpdate.toBsonDocument()); result = fileCollection.update(clientSession, queryBson, fileUpdate, null); @@ -981,7 +978,7 @@ OpenCGAResult count(ClientSession clientSession, Query query) public OpenCGAResult count(final Query query, final String user) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Bson bson = parseQuery(query, user); - logger.debug("File count: query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("File count: query : {}", bson.toBsonDocument()); return new OpenCGAResult<>(fileCollection.count(bson)); } @@ -1134,7 +1131,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu } qOptions = fixQueryOptions(qOptions); - logger.debug("File query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("File query: {}", bson.toBsonDocument()); if (!query.getBoolean(QueryParams.DELETED.key())) { return fileCollection.iterator(clientSession, bson, null, null, qOptions); } else { @@ -1417,9 +1414,7 @@ void removeSampleReferences(ClientSession clientSession, long studyUid, Sample s Bson bsonQuery = parseQuery(query); - logger.debug("Sample references extraction. Query: {}, update: {}", - bsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Sample references extraction. Query: {}, update: {}", bsonQuery.toBsonDocument(), update.toBsonDocument()); DataResult result = fileCollection.update(clientSession, bsonQuery, update, multi); logger.debug("Sample '" + sample.getId() + "' references removed from " + result.getNumUpdated() + " out of " + result.getNumMatches() + " files"); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java index 3b30317d6b6..1a074a17331 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Updates; @@ -296,8 +295,7 @@ public OpenCGAResult count(Query query, String user) OpenCGAResult count(ClientSession clientSession, Query query, String user) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Bson bson = parseQuery(query, user); - logger.debug("Individual count: query : {}, dbTime: {}", bson.toBsonDocument(Document.class, - MongoClient.getDefaultCodecRegistry())); + logger.debug("Individual count: query : {}, dbTime: {}", bson.toBsonDocument()); return new OpenCGAResult<>(individualCollection.count(clientSession, bson)); } @@ -416,9 +414,7 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Individual indi if (!individualUpdate.isEmpty()) { Bson finalQuery = parseQuery(tmpQuery); - logger.debug("Individual update: query : {}, update: {}", - finalQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - individualUpdate.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Individual update: query : {}, update: {}", finalQuery.toBsonDocument(), individualUpdate.toBsonDocument()); result = individualCollection.update(clientSession, finalQuery, individualUpdate, new QueryOptions("multi", true)); @@ -1024,9 +1020,8 @@ OpenCGAResult privateDelete(ClientSession clientSession, Document indivi ); Document update = familyDBAdaptor.parseAndValidateUpdateParams(clientSession, params, null).toFinalUpdateDocument(); - logger.debug("Remove individual references from family: Query: {}, update: {}", - bsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Remove individual references from family: Query: {}, update: {}", bsonQuery.toBsonDocument(), + update.toBsonDocument()); DataResult result = familyDBAdaptor.getFamilyCollection().update(clientSession, bsonQuery, update, new QueryOptions(MongoDBCollection.MULTI, true)); logger.debug("Families found: {}, families updated: {}", result.getNumMatches(), result.getNumUpdated()); @@ -1203,7 +1198,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu qOptions = filterOptions(qOptions, FILTER_ROUTE_INDIVIDUALS); fixAclProjection(qOptions); - logger.debug("Individual get: query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Individual get: query : {}", bson.toBsonDocument()); MongoDBCollection collection = getQueryCollection(query, individualCollection, archiveIndividualCollection, deletedIndividualCollection); return collection.iterator(clientSession, bson, null, null, qOptions); @@ -1690,10 +1685,8 @@ void removeSampleReferences(ClientSession clientSession, long studyUid, long sam Bson bsonQuery = parseQuery(query); versionedMongoDBAdaptor.update(clientSession, bsonQuery, () -> { QueryOptions multi = new QueryOptions(MongoDBCollection.MULTI, true); - - logger.debug("Sample references extraction. Query: {}, update: {}", - bsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Sample references extraction. Query: {}, update: {}", bsonQuery.toBsonDocument(), + update.toBsonDocument()); DataResult updateResult = individualCollection.update(clientSession, bsonQuery, update, multi); logger.debug("Sample uid '" + sampleUid + "' references removed from " + updateResult.getNumUpdated() + " out of " + updateResult.getNumMatches() + " individuals"); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java index 3bface77ed6..cf2b0a64eaa 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/InterpretationMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; @@ -333,7 +332,7 @@ public OpenCGAResult count(Query query) throws CatalogDBException { public OpenCGAResult count(ClientSession clientSession, Query query) throws CatalogDBException { Bson bson = parseQuery(query); - logger.debug("Interpretation count: query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Interpretation count: query : {}", bson.toBsonDocument()); return new OpenCGAResult<>(interpretationCollection.count(clientSession, bson)); } @@ -695,8 +694,7 @@ OpenCGAResult update(ClientSession clientSession, Interpretation interpretation.setVersion(interpretation.getVersion() + 1); updateClinicalAnalysisInterpretationReference(clientSession, interpretation, clinicalAuditList); if (!updateOperation.isEmpty()) { - logger.debug("Update interpretation. Query: {}, Update: {}", bsonQuery.toBsonDocument(Document.class, - MongoClient.getDefaultCodecRegistry()), updateDocument); + logger.debug("Update interpretation. Query: {}, Update: {}", bsonQuery.toBsonDocument(), updateDocument); update = interpretationCollection.update(clientSession, bsonQuery, updateOperation, null); if (update.getNumMatches() == 0) { @@ -710,8 +708,7 @@ OpenCGAResult update(ClientSession clientSession, Interpretation Bson nestedBsonQuery = parseQuery(nestedDocument.getQuery() .append(QueryParams.UID.key(), interpretation.getUid())); logger.debug("Update nested element from interpretation. Query: {}, Update: {}", - nestedBsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - nestedDocument.getSet()); + nestedBsonQuery.toBsonDocument(), nestedDocument.getSet()); update = interpretationCollection.update(clientSession, nestedBsonQuery, nestedDocument.getSet(), null); @@ -731,8 +728,7 @@ OpenCGAResult update(ClientSession clientSession, Interpretation UpdateDocument updateStatsDocument = parseAndValidateUpdateParams(clientSession, new ObjectMap(QueryParams.STATS.key(), stats), iQuery, QueryOptions.empty()); - logger.debug("Update interpretation stats. Query: {}, Update: {}", - bsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), + logger.debug("Update interpretation stats. Query: {}, Update: {}", bsonQuery.toBsonDocument(), updateStatsDocument.toFinalUpdateDocument()); DataResult statsUpdate = interpretationCollection.update(clientSession, bsonQuery, @@ -947,7 +943,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu qOptions = filterQueryOptions(qOptions, Arrays.asList(QueryParams.ID.key(), QueryParams.UUID.key(), QueryParams.UID.key(), QueryParams.VERSION.key(), QueryParams.CLINICAL_ANALYSIS_ID.key())); - logger.debug("Interpretation query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Interpretation query : {}", bson.toBsonDocument()); MongoDBCollection collection = getQueryCollection(query, interpretationCollection, archiveInterpretationCollection, deleteInterpretationCollection); return collection.iterator(clientSession, bson, null, null, qOptions); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java index 4087d70b2c8..48bedd2acaf 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import org.apache.commons.lang3.NotImplementedException; @@ -207,7 +206,7 @@ OpenCGAResult count(ClientSession clientSession, Query query) public OpenCGAResult count(Query query, String user) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Bson bson = parseQuery(query, QueryOptions.empty(), user); - logger.debug("Job count: query : {}, dbTime: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Job count: query : {}, dbTime: {}", bson.toBsonDocument()); return new OpenCGAResult<>(jobCollection.count(bson)); } @@ -281,9 +280,7 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Job job, Object .append(QueryParams.UID.key(), job.getUid()); Bson finalQuery = parseQuery(tmpQuery, options); - logger.debug("Job update: query : {}, update: {}", - finalQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - jobParameters.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Job update: query : {}, update: {}", finalQuery.toBsonDocument(), jobParameters.toBsonDocument()); DataResult result = jobCollection.update(clientSession, finalQuery, jobParameters, null); if (result.getNumMatches() == 0) { @@ -638,7 +635,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu Bson bson = parseQuery(query, options, user); - logger.debug("Job get: query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Job get: query : {}", bson.toBsonDocument()); if (!query.getBoolean(QueryParams.DELETED.key())) { return jobCollection.iterator(clientSession, bson, null, null, qOptions); } else { @@ -755,9 +752,8 @@ void removeFileReferences(ClientSession clientSession, long studyUid, long fileU document.getPush().put(prefix + Constants.JOB_DELETED_INPUT_FILES, file); Document updateDocument = document.toFinalUpdateDocument(); - logger.debug("Removing file from job '{}' field. Query: {}, Update: {}", QueryParams.INPUT.key(), - query.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - updateDocument.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Removing file from job '{}' field. Query: {}, Update: {}", QueryParams.INPUT.key(), query.toBsonDocument(), + updateDocument.toBsonDocument()); DataResult result = jobCollection.update(clientSession, query, updateDocument, QueryOptions.empty()); logger.debug("File '{}' removed from {} jobs", fileUid, result.getNumUpdated()); @@ -771,9 +767,8 @@ void removeFileReferences(ClientSession clientSession, long studyUid, long fileU document.getPush().put(prefix + Constants.JOB_DELETED_OUTPUT_FILES, file); updateDocument = document.toFinalUpdateDocument(); - logger.debug("Removing file from job '{}' field. Query: {}, Update: {}", QueryParams.OUTPUT.key(), - query.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - updateDocument.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Removing file from job '{}' field. Query: {}, Update: {}", QueryParams.OUTPUT.key(), query.toBsonDocument(), + updateDocument.toBsonDocument()); result = jobCollection.update(clientSession, query, updateDocument, QueryOptions.empty()); logger.debug("File '{}' removed from {} jobs", fileUid, result.getNumUpdated()); @@ -786,9 +781,8 @@ void removeFileReferences(ClientSession clientSession, long studyUid, long fileU document.getSet().put(prefix + Constants.JOB_DELETED_OUTPUT_DIRECTORY, file); updateDocument = document.toFinalUpdateDocument(); - logger.debug("Removing file from job '{}' field. Query: {}, Update: {}", QueryParams.OUT_DIR.key(), - query.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - updateDocument.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Removing file from job '{}' field. Query: {}, Update: {}", QueryParams.OUT_DIR.key(), query.toBsonDocument(), + updateDocument.toBsonDocument()); result = jobCollection.update(clientSession, query, updateDocument, QueryOptions.empty()); logger.debug("File '{}' removed from {} jobs", fileUid, result.getNumUpdated()); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java index 49ef6ed77f4..1a98f5a99a9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptor.java @@ -81,7 +81,7 @@ public long getNewAutoIncrementId(ClientSession clientSession, String field) { / return result.getResults().get(0).getLong(field); } - public void createIndexes(boolean wholeIndexes) { + public void createIndexes() { InputStream resourceAsStream = getClass().getResourceAsStream("/catalog-indexes.txt"); ObjectMapper objectMapper = getDefaultObjectMapper(); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(resourceAsStream)); @@ -99,10 +99,7 @@ public void createIndexes(boolean wholeIndexes) { Map myIndexes = new HashMap<>(); myIndexes.put("fields", new ObjectMap((Map) hashMap.get("fields"))); myIndexes.put("options", new ObjectMap((Map) hashMap.getOrDefault("options", Collections.emptyMap()))); - - if (wholeIndexes || myIndexes.get("options").getBoolean("unique")) { - indexes.get(collection).add(myIndexes); - } + indexes.get(collection).add(myIndexes); } } catch (IOException e) { e.printStackTrace(); @@ -179,20 +176,20 @@ public void clean(List collections) { } } - public void initializeMetaCollection(Configuration configuration) throws CatalogException { + public void initializeMetaCollection(Admin admin) throws CatalogException { Metadata metadata = new Metadata().setIdCounter(0L).setVersion(VERSION); // Ensure JWT secret key is long and secure - String secretKey = configuration.getAdmin().getSecretKey(); + String secretKey = admin.getSecretKey(); if (StringUtils.isEmpty(secretKey)) { throw new CatalogDBException("Missing JWT secret key"); } else { - JwtUtils.validateJWTKey(configuration.getAdmin().getAlgorithm(), secretKey); + JwtUtils.validateJWTKey(admin.getAlgorithm(), secretKey); } Document metadataObject = getMongoDBDocument(metadata, "Metadata"); metadataObject.put(ID, MongoDBAdaptorFactory.METADATA_OBJECT_ID); - Document adminDocument = getMongoDBDocument(configuration.getAdmin(), "Admin"); + Document adminDocument = getMongoDBDocument(admin, "Admin"); metadataObject.put("admin", adminDocument); metadataObject.put("_fullVersion", new Document() .append("version", 20003) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java index e92c4793d4a..05acf9aed2b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.mongodb.BasicDBObject; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.StopWatch; import org.bson.Document; import org.opencb.commons.datastore.core.DataStoreServerAddress; import org.opencb.commons.datastore.core.ObjectMap; @@ -30,11 +31,13 @@ import org.opencb.opencga.catalog.db.api.MigrationDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.core.config.Admin; import org.opencb.opencga.core.config.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; +import java.util.concurrent.TimeUnit; import static org.opencb.opencga.core.common.JacksonUtils.getDefaultObjectMapper; @@ -43,42 +46,6 @@ */ public class MongoDBAdaptorFactory implements DBAdaptorFactory { - private final List COLLECTIONS_LIST = Arrays.asList( - USER_COLLECTION, - STUDY_COLLECTION, - FILE_COLLECTION, - JOB_COLLECTION, - SAMPLE_COLLECTION, - INDIVIDUAL_COLLECTION, - COHORT_COLLECTION, - PANEL_COLLECTION, - FAMILY_COLLECTION, - CLINICAL_ANALYSIS_COLLECTION, - INTERPRETATION_COLLECTION, - - SAMPLE_ARCHIVE_COLLECTION, - INDIVIDUAL_ARCHIVE_COLLECTION, - FAMILY_ARCHIVE_COLLECTION, - PANEL_ARCHIVE_COLLECTION, - INTERPRETATION_ARCHIVE_COLLECTION, - - DELETED_USER_COLLECTION, - DELETED_STUDY_COLLECTION, - DELETED_FILE_COLLECTION, - DELETED_JOB_COLLECTION, - DELETED_SAMPLE_COLLECTION, - DELETED_INDIVIDUAL_COLLECTION, - DELETED_COHORT_COLLECTION, - DELETED_PANEL_COLLECTION, - DELETED_FAMILY_COLLECTION, - DELETED_CLINICAL_ANALYSIS_COLLECTION, - DELETED_INTERPRETATION_COLLECTION, - - MIGRATION_COLLECTION, - METADATA_COLLECTION, - AUDIT_COLLECTION - ); - public static final String USER_COLLECTION = "user"; public static final String STUDY_COLLECTION = "study"; public static final String FILE_COLLECTION = "file"; @@ -135,6 +102,43 @@ public class MongoDBAdaptorFactory implements DBAdaptorFactory { public static final String METADATA_COLLECTION = "metadata"; public static final String MIGRATION_COLLECTION = "migration"; public static final String AUDIT_COLLECTION = "audit"; + + public static final List COLLECTIONS_LIST = Arrays.asList( + USER_COLLECTION, + STUDY_COLLECTION, + FILE_COLLECTION, + JOB_COLLECTION, + SAMPLE_COLLECTION, + INDIVIDUAL_COLLECTION, + COHORT_COLLECTION, + PANEL_COLLECTION, + FAMILY_COLLECTION, + CLINICAL_ANALYSIS_COLLECTION, + INTERPRETATION_COLLECTION, + + SAMPLE_ARCHIVE_COLLECTION, + INDIVIDUAL_ARCHIVE_COLLECTION, + FAMILY_ARCHIVE_COLLECTION, + PANEL_ARCHIVE_COLLECTION, + INTERPRETATION_ARCHIVE_COLLECTION, + + DELETED_USER_COLLECTION, + DELETED_STUDY_COLLECTION, + DELETED_FILE_COLLECTION, + DELETED_JOB_COLLECTION, + DELETED_SAMPLE_COLLECTION, + DELETED_INDIVIDUAL_COLLECTION, + DELETED_COHORT_COLLECTION, + DELETED_PANEL_COLLECTION, + DELETED_FAMILY_COLLECTION, + DELETED_CLINICAL_ANALYSIS_COLLECTION, + DELETED_INTERPRETATION_COLLECTION, + + MIGRATION_COLLECTION, + METADATA_COLLECTION, + AUDIT_COLLECTION + ); + static final String METADATA_OBJECT_ID = "METADATA"; private final MongoDataStoreManager mongoManager; private final MongoDBConfiguration configuration; @@ -176,6 +180,7 @@ public MongoDBAdaptorFactory(Configuration catalogConfiguration) throws CatalogD .setUserPassword( catalogConfiguration.getCatalog().getDatabase().getUser(), catalogConfiguration.getCatalog().getDatabase().getPassword()) + .setConnectionsPerHost(200) .setServerAddress(dataStoreServerAddresses) .load(catalogConfiguration.getCatalog().getDatabase().getOptions()) .build(); @@ -189,7 +194,7 @@ public MongoDBAdaptorFactory(Configuration catalogConfiguration) throws CatalogD } @Override - public void installCatalogDB(Configuration configuration) throws CatalogException { + public void createAllCollections(Configuration configuration) throws CatalogException { // TODO: Check META object does not exist. Use {@link isCatalogDBReady} // TODO: Check all collections do not exists, or are empty // TODO: Catch DuplicatedKeyException while inserting META object @@ -200,12 +205,18 @@ public void installCatalogDB(Configuration configuration) throws CatalogExceptio + StringUtils.join(mongoDataStore.getCollectionNames()) + ".\nPlease, remove the database or choose a different one."); } COLLECTIONS_LIST.forEach(mongoDataStore::createCollection); - metaDBAdaptor.initializeMetaCollection(configuration); } @Override - public void createIndexes(boolean wholeIndexes) { - metaDBAdaptor.createIndexes(wholeIndexes); + public void initialiseMetaCollection(Admin admin) throws CatalogException { + metaDBAdaptor.initializeMetaCollection(admin); + } + + @Override + public void createIndexes() { + StopWatch stopWatch = StopWatch.createStarted(); + metaDBAdaptor.createIndexes(); + logger.info("Creating all indexes took {} milliseconds", stopWatch.getTime(TimeUnit.MILLISECONDS)); } @Override diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java index 59bdfdf4055..35b290c064d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/PanelMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; @@ -270,7 +269,7 @@ public OpenCGAResult count(final Query query, final String user) OpenCGAResult count(ClientSession clientSession, final Query query, final String user) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Bson bson = parseQuery(query, user); - logger.debug("Panel count: query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Panel count: query : {}", bson.toBsonDocument()); return new OpenCGAResult<>(panelCollection.count(clientSession, bson)); } @@ -345,9 +344,7 @@ private OpenCGAResult privateUpdate(ClientSession clientSession, Panel p Bson finalQuery = parseQuery(tmpQuery); return versionedMongoDBAdaptor.update(clientSession, finalQuery, () -> { - logger.debug("Panel update: query : {}, update: {}", - finalQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - panelUpdate.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Panel update: query : {}, update: {}", finalQuery.toBsonDocument(), panelUpdate.toBsonDocument()); DataResult result = panelCollection.update(clientSession, finalQuery, new Document("$set", panelUpdate), new QueryOptions("multi", true)); @@ -613,7 +610,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu } Bson bson = parseQuery(finalQuery, user); - logger.debug("Panel query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Panel query: {}", bson.toBsonDocument()); MongoDBCollection collection = getQueryCollection(query, panelCollection, panelArchiveCollection, deletedPanelCollection); return collection.iterator(clientSession, bson, null, null, qOptions); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java index acfaabf7664..72cf25e4136 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; @@ -24,8 +23,6 @@ import com.mongodb.client.model.Updates; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; -import org.bson.BsonDocument; -import org.bson.BsonString; import org.bson.Document; import org.bson.conversions.Bson; import org.opencb.commons.datastore.core.*; @@ -143,9 +140,7 @@ Project insert(ClientSession clientSession, Project project, String userId) //Update object Bson query = Filters.eq(UserDBAdaptor.QueryParams.ID.key(), userId); - logger.debug("Inserting project. Query: {}, update: {}", - query.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Inserting project. Query: {}, update: {}", query.toBsonDocument(), update.toBsonDocument()); userCollection.update(clientSession, query, update, null); return project; @@ -198,12 +193,12 @@ public long getId(final String userId, final String projectIdStr) throws Catalog projectId = projectId.split("@", 2)[1]; } - DataResult queryResult = userCollection.find( - new BsonDocument(UserDBAdaptor.QueryParams.PROJECTS_ID.key(), new BsonString(projectId)) - .append(UserDBAdaptor.QueryParams.ID.key(), new BsonString(userId)), - Projections.fields(Projections.include(UserDBAdaptor.QueryParams.PROJECTS_UID.key()), - Projections.elemMatch("projects", Filters.eq(QueryParams.ID.key(), projectId))), - null); + Bson filter = Filters.and( + Filters.eq(UserDBAdaptor.QueryParams.PROJECTS_ID.key(), projectId), + Filters.eq(UserDBAdaptor.QueryParams.ID.key(), userId) + ); + Bson projection = Projections.elemMatch("projects", Filters.eq(QueryParams.ID.key(), projectId)); + DataResult queryResult = userCollection.find(filter, projection, null); User user = parseUser(queryResult); if (user == null || user.getProjects().isEmpty()) { return -1; @@ -316,9 +311,7 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Project project Query tmpQuery = new Query(QueryParams.UID.key(), project.getUid()); Bson finalQuery = parseQuery(tmpQuery); - logger.debug("Update project. Query: {}, update: {}", - finalQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - updates.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Update project. Query: {}, update: {}", finalQuery.toBsonDocument(), updates.toBsonDocument()); DataResult result = userCollection.update(clientSession, finalQuery, updates, null); if (result.getNumMatches() == 0) { @@ -505,9 +498,8 @@ OpenCGAResult privateDelete(ClientSession clientSession, Project projec Bson bsonQuery = parseQuery(studyQuery); Document updateDocument = getDocumentUpdateParams(updateParams); - logger.debug("Delete project {}: Query: {}, update: {}", project.getId(), - bsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - updateDocument.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Delete project {}: Query: {}, update: {}", project.getId(), bsonQuery.toBsonDocument(), + updateDocument.toBsonDocument()); DataResult result = userCollection.update(clientSession, bsonQuery, updateDocument, QueryOptions.empty()); if (result.getNumMatches() == 0) { @@ -793,7 +785,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu } for (Bson aggregate : aggregates) { - logger.debug("Get project: Aggregate : {}", aggregate.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Get project: Aggregate : {}", aggregate.toBsonDocument()); } QueryOptions qOptions = new QueryOptions(); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index 05b165048c4..b037f774556 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -17,7 +17,6 @@ package org.opencb.opencga.catalog.db.mongodb; import com.fasterxml.jackson.core.JsonProcessingException; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; @@ -65,7 +64,7 @@ import java.util.function.Consumer; import java.util.function.UnaryOperator; -import static org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor.QueryParams.*; +import static org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor.QueryParams.MODIFICATION_DATE; import static org.opencb.opencga.catalog.db.mongodb.AuthorizationMongoDBUtils.filterAnnotationSets; import static org.opencb.opencga.catalog.db.mongodb.AuthorizationMongoDBUtils.getQueryForAuthorisedEntries; import static org.opencb.opencga.catalog.db.mongodb.MongoDBUtils.*; @@ -321,7 +320,7 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Document sample // Perform the update DataResult result = updateAnnotationSets(clientSession, sampleUid, parameters, variableSetList, queryOptions, true); - UpdateDocument updateParams = parseAndValidateUpdateParams(clientSession, parameters, tmpQuery, queryOptions); + UpdateDocument updateParams = parseAndValidateUpdateParams(clientSession, studyUid, parameters, tmpQuery, queryOptions); Document sampleUpdate = updateParams.toFinalUpdateDocument(); if (sampleUpdate.isEmpty() && result.getNumUpdated() == 0) { @@ -335,9 +334,7 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Document sample if (!sampleUpdate.isEmpty()) { Bson finalQuery = parseQuery(tmpQuery); - logger.debug("Sample update: query : {}, update: {}", - finalQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - sampleUpdate.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Sample update: query : {}, update: {}", finalQuery.toBsonDocument(), sampleUpdate.toBsonDocument()); result = sampleCollection.update(clientSession, finalQuery, sampleUpdate, new QueryOptions("multi", true)); if (updateParams.getSet().containsKey(PRIVATE_INDIVIDUAL_UID)) { @@ -386,7 +383,7 @@ void updateIndividualFromSampleCollection(ClientSession clientSession, long stud } ObjectMap params = new ObjectMap(QueryParams.INDIVIDUAL_ID.key(), individualId); - Document update = parseAndValidateUpdateParams(clientSession, params, null, QueryOptions.empty()).toFinalUpdateDocument(); + Document update = parseAndValidateUpdateParams(clientSession, studyId, params, null, QueryOptions.empty()).toFinalUpdateDocument(); Bson query = parseQuery(new Query() .append(QueryParams.STUDY_UID.key(), studyId) .append(QueryParams.UID.key(), sampleUids)); @@ -537,7 +534,8 @@ private void updateSampleFromIndividualCollection(ClientSession clientSession, S individualDBAdaptor.getCollection().update(clientSession, bsonQuery, update, null); } - UpdateDocument parseAndValidateUpdateParams(ClientSession clientSession, ObjectMap parameters, Query query, QueryOptions queryOptions) + UpdateDocument parseAndValidateUpdateParams(ClientSession clientSession, long studyUid, ObjectMap parameters, Query query, + QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { UpdateDocument document = new UpdateDocument(); @@ -625,13 +623,18 @@ UpdateDocument parseAndValidateUpdateParams(ClientSession clientSession, ObjectM if (StringUtils.isNotEmpty(individualId)) { // Look for the individual uid - Query indQuery = new Query(IndividualDBAdaptor.QueryParams.ID.key(), individualId); + Query indQuery = new Query() + .append(IndividualDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) + .append(IndividualDBAdaptor.QueryParams.ID.key(), individualId); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, IndividualDBAdaptor.QueryParams.UID.key()); OpenCGAResult individualDataResult = individualDBAdaptor.get(clientSession, indQuery, options); if (individualDataResult.getNumResults() == 0) { throw new CatalogDBException("Cannot update " + QueryParams.INDIVIDUAL_ID.key() + " for sample. Individual '" + individualId + "' not found."); + } else if (individualDataResult.getNumResults() > 1) { + throw new CatalogDBException("Cannot update " + QueryParams.INDIVIDUAL_ID.key() + " for sample. More than one" + + " Individual '" + individualId + "' found."); } document.getSet().put(QueryParams.INDIVIDUAL_ID.key(), individualId); @@ -824,18 +827,15 @@ public OpenCGAResult count(Query query, String user) Bson count = Aggregates.count("count"); - logger.debug("Sample count aggregation: {} -> {} -> {} -> {}", - match.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - lookup.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - individualMatch.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - count.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Sample count aggregation: {} -> {} -> {} -> {}", match.toBsonDocument(), lookup.toBsonDocument(), + individualMatch.toBsonDocument(), count.toBsonDocument()); DataResult aggregate = sampleCollection.aggregate(Arrays.asList(match, lookup, individualMatch, count), QueryOptions.empty()); long numResults = aggregate.getNumResults() == 0 ? 0 : ((int) aggregate.first().get("count")); return new OpenCGAResult<>(aggregate.getTime(), Collections.emptyList(), 1, Collections.singletonList(numResults), 1); } else { - logger.debug("Sample count query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Sample count query: {}", bson.toBsonDocument()); return new OpenCGAResult<>(sampleCollection.count(bson)); } } @@ -930,9 +930,8 @@ void removeFileReferences(ClientSession clientSession, long studyUid, String fil .append(QueryParams.FILE_IDS.key(), fileId); Bson bsonQuery = parseQuery(query); - logger.debug("Removing file from sample '{}' field. Query: {}, Update: {}", QueryParams.FILE_IDS.key(), - bsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - updateDocument.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Removing file from sample '{}' field. Query: {}, Update: {}", QueryParams.FILE_IDS.key(), bsonQuery.toBsonDocument(), + updateDocument.toBsonDocument()); versionedMongoDBAdaptor.update(clientSession, bsonQuery, () -> { DataResult result = sampleCollection.update(clientSession, bsonQuery, updateDocument, new QueryOptions("multi", true)); @@ -1148,7 +1147,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu Bson bson = parseQuery(finalQuery, user); MongoDBCollection collection = getQueryCollection(finalQuery, sampleCollection, archiveSampleCollection, deletedSampleCollection); - logger.debug("Sample query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Sample query: {}", bson.toBsonDocument()); return collection.iterator(clientSession, bson, null, null, qOptions); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java index 6632b14f0e6..278035f3a4d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Aggregates; import com.mongodb.client.model.Filters; @@ -693,9 +692,7 @@ public OpenCGAResult markDeletedPermissionRule(long studyId, Enu Document update = new Document("$set", new Document(QueryParams.PERMISSION_RULES.key() + "." + entry + ".$.id", newPermissionRuleId)); - logger.debug("Mark permission rule for deletion: Query {}, Update {}", - query.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Mark permission rule for deletion: Query {}, Update {}", query.toBsonDocument(), update.toBsonDocument()); DataResult result = studyCollection.update(query, update, QueryOptions.empty()); if (result.getNumMatches() == 0) { @@ -1435,9 +1432,7 @@ OpenCGAResult privateUpdate(ClientSession clientSession, Study study, Ob Query tmpQuery = new Query(QueryParams.UID.key(), study.getUid()); Bson finalQuery = parseQuery(tmpQuery); - logger.debug("Update study. Query: {}, update: {}", - finalQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - updates.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Update study. Query: {}, update: {}", finalQuery.toBsonDocument(), updates.toBsonDocument()); DataResult result = studyCollection.update(clientSession, finalQuery, updates, null); if (result.getNumMatches() == 0) { @@ -1769,7 +1764,7 @@ private MongoDBIterator getMongoCursor(ClientSession clientSession, Qu Bson bson = parseQuery(query); - logger.debug("Study native get: query : {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Study native get: query : {}", bson.toBsonDocument()); if (!query.getBoolean(QueryParams.DELETED.key())) { return studyCollection.iterator(clientSession, bson, null, null, qOptions); } else { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java index 9ba9d3c91a4..a25d2eabd10 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/UserMongoDBAdaptor.java @@ -16,7 +16,6 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Updates; @@ -290,7 +289,7 @@ public OpenCGAResult count(Query query) throws CatalogDBException { OpenCGAResult count(ClientSession clientSession, Query query) throws CatalogDBException { Bson bsonDocument = parseQuery(query); - logger.debug("User count: {}", bsonDocument.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("User count: {}", bsonDocument.toBsonDocument()); return new OpenCGAResult<>(userCollection.count(clientSession, bsonDocument)); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java index fd463503d3f..abafe45bf38 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/VersionedMongoDBAdaptor.java @@ -1,6 +1,5 @@ package org.opencb.opencga.catalog.db.mongodb; -import com.mongodb.MongoClient; import com.mongodb.client.ClientSession; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Updates; @@ -179,17 +178,15 @@ protected T update(ClientSession session, Bson sourceQuery, VersionedMode Filters.eq(VERSION, version) ); // Update previous version - logger.debug("Updating previous version: query : {}, update: {}", - bsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - archiveCollectionUpdate.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Updating previous version: query : {}, update: {}", bsonQuery.toBsonDocument(), + archiveCollectionUpdate.toBsonDocument()); archiveCollection.update(session, bsonQuery, new Document("$set", archiveCollectionUpdate), QueryOptions.empty()); // Add current transaction id to the document so we don't enter here twice in the same transaction collectionUpdate.put(PRIVATE_TRANSACTION_ID, uuid); // Update current version - logger.debug("Updating current version: query : {}, update: {}", - bsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()), - collectionUpdate.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Updating current version: query : {}, update: {}", bsonQuery.toBsonDocument(), + collectionUpdate.toBsonDocument()); collection.update(session, bsonQuery, new Document("$set", collectionUpdate), QueryOptions.empty()); } } @@ -218,8 +215,7 @@ protected T update(ClientSession session, Bson sourceQuery, VersionedMode Filters.eq(PRIVATE_UID, fixedResult.get(PRIVATE_UID)), Filters.eq(VERSION, fixedResult.get(VERSION)) ); - logger.debug("Copying current document to archive: query : {}", - tmpBsonQuery.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("Copying current document to archive: query : {}", tmpBsonQuery.toBsonDocument()); archiveCollection.update(session, tmpBsonQuery, fixedResult, upsertOptions); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index 95815117d12..56dcd11f325 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -171,7 +171,12 @@ public boolean existsCatalogDB() { return catalogDBAdaptorFactory.isCatalogDBReady(); } - public void installCatalogDB(String secretKey, String password, String email, String organization, boolean force, boolean wholeIndexes) + public void installCatalogDB(String secretKey, String password, String email, String organization, boolean force) + throws CatalogException { + installCatalogDB(secretKey, password, email, organization, force, false); + } + + public void installCatalogDB(String secretKey, String password, String email, String organization, boolean force, boolean test) throws CatalogException { if (existsCatalogDB()) { if (force) { @@ -193,16 +198,17 @@ public void installCatalogDB(String secretKey, String password, String email, St try { logger.info("Installing database {} in {}", getCatalogDatabase(), configuration.getCatalog().getDatabase().getHosts()); - installCatalogDB(secretKey, password, email, organization); + privateInstall(secretKey, password, email, organization, test); String token = userManager.loginAsAdmin(password).getToken(); - installIndexes(token, wholeIndexes); + installIndexes(token); } catch (Exception e) { clearCatalog(); throw e; } } - private void installCatalogDB(String secretKey, String password, String email, String organization) throws CatalogException { + private void privateInstall(String secretKey, String password, String email, String organization, boolean test) + throws CatalogException { if (existsCatalogDB()) { throw new CatalogException("Nothing to install. There already exists a catalog database"); } @@ -212,10 +218,12 @@ private void installCatalogDB(String secretKey, String password, String email, S ParamUtils.checkParameter(secretKey, "secretKey"); ParamUtils.checkParameter(password, "password"); JwtUtils.validateJWTKey(configuration.getAdmin().getAlgorithm(), secretKey); - configuration.getAdmin().setSecretKey(secretKey); - catalogDBAdaptorFactory.installCatalogDB(configuration); + if (!test) { + catalogDBAdaptorFactory.createAllCollections(configuration); + } + catalogDBAdaptorFactory.initialiseMetaCollection(configuration.getAdmin()); catalogIOManager.createDefaultOpenCGAFolders(); User user = new User(OPENCGA, new Account().setType(Account.AccountType.ADMINISTRATOR).setExpirationDate("")) @@ -232,14 +240,10 @@ private void installCatalogDB(String secretKey, String password, String email, S } public void installIndexes(String token) throws CatalogException { - installIndexes(token, false); - } - - public void installIndexes(String token, boolean wholeIndexes) throws CatalogException { if (!OPENCGA.equals(userManager.getUserId(token))) { throw new CatalogAuthorizationException("Only the admin can install new indexes"); } - catalogDBAdaptorFactory.createIndexes(wholeIndexes); + catalogDBAdaptorFactory.createIndexes(); } public void deleteCatalogDB(String password) throws CatalogException { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index b891b6c4d80..bf169d47efb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -567,6 +567,9 @@ public OpenCGAResult create(String studyStr, FileCreateParams createParams } if (createParams.getType().equals(File.Type.FILE)) { + if (path.endsWith("/")) { + throw new CatalogException("Provided path points to a folder but the type was set to FILE. Please check your input."); + } // Ensure path is a full path pointing to a file, not a folder String[] split = path.split("/"); String fileName = split[split.length - 1]; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java index 1806148ed97..1308019a28e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java @@ -309,7 +309,10 @@ private void validateProjectForCreation(Project project, User user) throws Catal ProjectDBAdaptor.QueryParams.MODIFICATION_DATE.key())); project.setModificationDate(TimeUtils.getTime()); project.setCellbase(ParamUtils.defaultObject(project.getCellbase(), - new CellBaseConfiguration(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION))); + new CellBaseConfiguration( + ParamConstants.CELLBASE_URL, + ParamConstants.CELLBASE_VERSION, + ParamConstants.CELLBASE_DATA_RELEASE))); project.setCurrentRelease(1); project.setInternal(ProjectInternal.init()); project.setAttributes(ParamUtils.defaultObject(project.getAttributes(), HashMap::new)); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java index a0b7ca0539f..1865880123a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/templates/TemplateManager.java @@ -135,9 +135,7 @@ public void validate(TemplateManifest manifest) throws CatalogException { versionShort = version; } String templateVersion = manifest.getConfiguration().getVersion(); - if (!version.equals(templateVersion) && !versionShort.equals(templateVersion)) { - throw new IllegalArgumentException("Version mismatch! Expected " + templateVersion + " but found " + versionShort); - } + checkVersion(versionShort, templateVersion); // Study should already exist Study study = getStudy(manifest.getConfiguration().getProjectId(), manifest.getStudy().getId()); @@ -152,6 +150,25 @@ public void validate(TemplateManifest manifest) throws CatalogException { // } } + private void checkVersion(String opencgaVersion, String templateVersion) { + if (opencgaVersion.equals(templateVersion)) { + return; + } + String[] ocgaVersionSplit = opencgaVersion.split("\\."); + String[] templateVersionSplit = templateVersion.split("\\."); + + for (int i = 0; i < templateVersionSplit.length; i++) { + int ocgaV = Integer.parseInt(ocgaVersionSplit[i]); + int tplV = Integer.parseInt(templateVersionSplit[i]); + if (ocgaV < tplV) { + throw new IllegalArgumentException("Cannot use a template with version higher than the OpenCGA installation version. " + + "Template version: " + templateVersion + ", OpenCGA version: " + opencgaVersion); + } + } + logger.warn("Using a template version lower than the OpenCGA installation version. Some things may not work properly. " + + "Template version: " + templateVersion + ", OpenCGA version: " + opencgaVersion); + } + private Study getStudy(String projectId, String studyId) throws CatalogException { OpenCGAResult studyOpenCGAResult = catalogManager.getStudyManager().get(projectId + ":" + studyId, QueryOptions.empty(), token); @@ -228,7 +245,7 @@ private String addStudyMetadata(String projectId, TemplateStudy tmplStudy) throw } } } - if (CollectionUtils.isNotEmpty(tmplStudy.getAcl().getAcl())) { + if (tmplStudy.getAcl() != null && CollectionUtils.isNotEmpty(tmplStudy.getAcl().getAcl())) { // Set permissions for (AclEntry studyAclEntry : tmplStudy.getAcl().getAcl()) { logger.info("Setting permissions for '{}'", studyAclEntry.getMember()); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java index 3b539e3eb49..91708265ca4 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/CatalogDemo.java @@ -47,7 +47,7 @@ private CatalogDemo() { public static void createDemoDatabase(CatalogManager catalogManager, String adminPassword, boolean force) throws CatalogException { catalogManager.installCatalogDB(catalogManager.getConfiguration().getAdmin().getSecretKey(), adminPassword, "opencga@admin.com", - "", force, true); + "", force); try { populateDatabase(catalogManager); } catch (IOException e) { diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index a77b33e5fd9..996eeb67534 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -1,250 +1,250 @@ -{"collections": ["user"], "fields": {"id": 1}, "options": {"unique": true, "background": true}} -{"collections": ["user"], "fields": {"projects.uid": 1, "id": 1}, "options": {"unique": true, "background": true}} -{"collections": ["user"], "fields": {"projects.uuid": 1, "id": 1}, "options": {"unique": true, "background": true}} -{"collections": ["user"], "fields": {"projects.fqn": 1, "id": 1}, "options": {"unique": true, "background": true}} -{"collections": ["user"], "fields": {"projects.id": 1, "id": 1}, "options": {"unique": true, "background": true}} +{"collections": ["user"], "fields": {"id": 1}, "options": {"unique": true}} +{"collections": ["user"], "fields": {"projects.uid": 1, "id": 1}, "options": {"unique": true}} +{"collections": ["user"], "fields": {"projects.uuid": 1, "id": 1}, "options": {"unique": true}} +{"collections": ["user"], "fields": {"projects.fqn": 1, "id": 1}, "options": {"unique": true}} +{"collections": ["user"], "fields": {"projects.id": 1, "id": 1}, "options": {"unique": true}} -{"collections": ["study"], "fields": {"uid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["study"], "fields": {"uuid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["study"], "fields": {"fqn": 1}, "options": {"unique": true, "background": true}} -{"collections": ["study"], "fields": {"id": 1, "_project.uid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["study"], "fields": {"groups.id": 1, "uid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["study"], "fields": {"groups.userIds": 1, "uid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["study"], "fields": {"_acl": 1}, "options": {"background": true}} -{"collections": ["study"], "fields": {"_project.uid": 1}, "options": {"background": true}} -{"collections": ["study"], "fields": {"variableSets.id": 1, "uid": 1}, "options": {"unique": true, "background": true}} +{"collections": ["study"], "fields": {"uid": 1}, "options": {"unique": true}} +{"collections": ["study"], "fields": {"uuid": 1}, "options": {"unique": true}} +{"collections": ["study"], "fields": {"fqn": 1}, "options": {"unique": true}} +{"collections": ["study"], "fields": {"id": 1, "_project.uid": 1}, "options": {"unique": true}} +{"collections": ["study"], "fields": {"groups.id": 1, "uid": 1}, "options": {"unique": true}} +{"collections": ["study"], "fields": {"groups.userIds": 1, "uid": 1}, "options": {"unique": true}} +{"collections": ["study"], "fields": {"_acl": 1}, "options": {}} +{"collections": ["study"], "fields": {"_project.uid": 1}, "options": {}} +{"collections": ["study"], "fields": {"variableSets.id": 1, "uid": 1}, "options": {"unique": true}} -{"collections": ["job"], "fields": {"uuid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["job"], "fields": {"uid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["job"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["job"], "fields": {"tool.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"tool.type": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"userId": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"input.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"output.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"outDir.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"tags": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"visited": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"status.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"_priority": 1, "_creationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"_priority": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"studyUid": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["job"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {"background": true}} +{"collections": ["job"], "fields": {"uuid": 1}, "options": {"unique": true}} +{"collections": ["job"], "fields": {"uid": 1}, "options": {"unique": true}} +{"collections": ["job"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["job"], "fields": {"tool.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"tool.type": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"userId": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"input.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"output.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"outDir.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"tags": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"visited": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"_priority": 1, "_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"_priority": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} +{"collections": ["job"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} -{"collections": ["file"], "fields": {"uuid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["file"], "fields": {"uid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["file"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["file"], "fields": {"path": 1, "studyUid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["file"], "fields": {"name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"uri": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"_reverse": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"type": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"format": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"bioformat": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"tags": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"size": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"software.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"external": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"_samples.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"_samples.uuid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"jobId": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"status.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"internal.variant.index.status.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"studyUid": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"_as.as": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"_as.vs": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"_as.id": 1, "_as.value": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["file"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {"background": true}} +{"collections": ["file"], "fields": {"uuid": 1}, "options": {"unique": true}} +{"collections": ["file"], "fields": {"uid": 1}, "options": {"unique": true}} +{"collections": ["file"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["file"], "fields": {"path": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["file"], "fields": {"name": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"uri": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"_reverse": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"type": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"format": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"bioformat": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"tags": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"size": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"software.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"external": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"_samples.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"_samples.uuid": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"jobId": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"internal.variant.index.status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} +{"collections": ["file"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} +{"collections": ["file"], "fields": {"_as.as": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"_as.vs": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"_as.id": 1, "_as.value": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} +{"collections": ["file"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} -{"collections": ["sample", "sample_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"id": 1, "studyUid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"somatic": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"phenotypes.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"phenotypes.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"fileIds": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"cohortIds": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"individualId": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"processing.product.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"processing.preparationMethod": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"processing.extractionMethod": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"processing.labSampleId": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"collection.from.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"collection.method": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"collection.type": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"_as.as": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"_as.vs": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"_as.id": 1, "_as.value": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["sample", "sample_archive"], "fields": {"internal.rga.status": 1, "studyUid": 1}, "options": {"background": true}} +{"collections": ["sample", "sample_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["sample", "sample_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["sample", "sample_archive"], "fields": {"id": 1, "studyUid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["sample", "sample_archive"], "fields": {"somatic": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"phenotypes.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"phenotypes.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"fileIds": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"cohortIds": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"individualId": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"processing.product.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"processing.preparationMethod": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"processing.extractionMethod": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"processing.labSampleId": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"collection.from.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"collection.method": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"collection.type": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1, "_acl": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"_as.as": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"_as.vs": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"_as.id": 1, "_as.value": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"internal.rga.status": 1, "studyUid": 1}, "options": {}} -{"collections": ["individual", "individual_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"id": 1, "studyUid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"father.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"mother.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"familyIds": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"sex.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"karyotypicSex": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"ethnicity.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"dateOfBirth": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"lifeStatus": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"samples.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"phenotypes.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"phenotypes.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"disorders.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"disorders.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"population.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"population.subpopulation": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"_as.as": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"_as.vs": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"_as.id": 1, "_as.value": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["individual", "individual_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {"background": true}} +{"collections": ["individual", "individual_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["individual", "individual_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["individual", "individual_archive"], "fields": {"id": 1, "studyUid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["individual", "individual_archive"], "fields": {"name": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"father.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"mother.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"familyIds": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"sex.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"karyotypicSex": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"ethnicity.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"dateOfBirth": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"lifeStatus": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"samples.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"phenotypes.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"phenotypes.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"disorders.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"disorders.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"population.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"population.subpopulation": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1, "_acl": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"_as.as": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"_as.vs": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"_as.id": 1, "_as.value": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["cohort"], "fields": {"uuid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["cohort"], "fields": {"uid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["cohort"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["cohort"], "fields": {"name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"type": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"samples.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"numSamples": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"studyUid": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"_as.as": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"_as.vs": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"_as.id": 1, "_as.value": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"status.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["cohort"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {"background": true}} +{"collections": ["cohort"], "fields": {"uuid": 1}, "options": {"unique": true}} +{"collections": ["cohort"], "fields": {"uid": 1}, "options": {"unique": true}} +{"collections": ["cohort"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["cohort"], "fields": {"name": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"type": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"samples.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"numSamples": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"_as.as": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"_as.vs": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"_as.id": 1, "_as.value": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["cohort"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["family", "family_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["family", "family_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["family", "family_archive"], "fields": {"id": 1, "studyUid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["family", "family_archive"], "fields": {"name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"members.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"phenotypes.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"phenotypes.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"disorders.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"disorders.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"expectedSize": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"_as.as": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"_as.vs": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"_as.id": 1, "_as.value": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["family", "family_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {"background": true}} +{"collections": ["family", "family_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["family", "family_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["family", "family_archive"], "fields": {"id": 1, "studyUid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["family", "family_archive"], "fields": {"name": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"members.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"phenotypes.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"phenotypes.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"disorders.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"disorders.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"expectedSize": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1, "_acl": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"_as.as": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"_as.vs": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"_as.id": 1, "_as.value": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"_ias.as": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"_ias.vs": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["panel", "panel_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"id": 1, "studyUid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"tags": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"disorders.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"disorders.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"variants.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"genes.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"genes.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"source.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"source.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"regions.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"categories.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["panel", "panel_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {"background": true}} +{"collections": ["panel", "panel_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["panel", "panel_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["panel", "panel_archive"], "fields": {"id": 1, "studyUid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["panel", "panel_archive"], "fields": {"name": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"tags": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"disorders.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"disorders.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"variants.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"genes.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"genes.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"source.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"source.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"regions.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"categories.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1, "_acl": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"studyUid": 1, "release": 1, "_acl": 1}, "options": {}} +{"collections": ["panel", "panel_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["clinical"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["clinical"], "fields": {"uuid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["clinical"], "fields": {"uid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["clinical"], "fields": {"type": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"files.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"proband.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"proband.samples.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"family.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"family.members.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"family.members.samples.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"panels.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"panelLock": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"locked": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"_dueDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"disorder.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"disorder.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"priority.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"flags.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"qualityControl.summary": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"studyUid": 1, "_acl": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"status.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["clinical"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {"background": true}} +{"collections": ["clinical"], "fields": {"id": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["clinical"], "fields": {"uuid": 1}, "options": {"unique": true}} +{"collections": ["clinical"], "fields": {"uid": 1}, "options": {"unique": true}} +{"collections": ["clinical"], "fields": {"type": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"files.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"proband.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"proband.samples.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"family.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"family.members.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"family.members.samples.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"panels.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"panelLock": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"locked": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"_dueDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"disorder.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"disorder.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"priority.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"flags.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"qualityControl.summary": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true, "background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"id": 1, "version": 1, "studyUid": 1}, "options": {"unique": true, "background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"clinicalAnalysisId": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"method.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"panels.uid": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"primaryFindings.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"secondaryFindings.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "release": 1}, "options": {"background": true}} -{"collections": ["interpretation", "interpretation_archive"], "fields": {"locked": 1, "studyUid": 1}, "options": {"background": true}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"id": 1, "version": 1, "studyUid": 1}, "options": {"unique": true}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"clinicalAnalysisId": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"analyst.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"method.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"panels.uid": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"primaryFindings.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"secondaryFindings.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"internal.status.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "_lastOfVersion": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "_releaseFromVersion": 1, "_lastOfRelease": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"studyUid": 1, "release": 1}, "options": {}} +{"collections": ["interpretation", "interpretation_archive"], "fields": {"locked": 1, "studyUid": 1}, "options": {}} -{"collections": ["audit"], "fields": {"studyUuid": 1}, "options": {"background": true}} -{"collections": ["audit"], "fields": {"userId": 1, "studyUuid": 1}, "options": {"background": true}} -{"collections": ["audit"], "fields": {"action": 1, "studyUuid": 1}, "options": {"background": true}} -{"collections": ["audit"], "fields": {"resource": 1, "studyUuid": 1}, "options": {"background": true}} -{"collections": ["audit"], "fields": {"resourceId": 1, "studyUuid": 1}, "options": {"background": true}} -{"collections": ["audit"], "fields": {"resourceUuid": 1, "studyUuid": 1}, "options": {"background": true}} -{"collections": ["audit"], "fields": {"status.name": 1, "studyUuid": 1}, "options": {"background": true}} -{"collections": ["audit"], "fields": {"date": 1, "studyUuid": 1}, "options": {"background": true}} \ No newline at end of file +{"collections": ["audit"], "fields": {"studyUuid": 1}, "options": {}} +{"collections": ["audit"], "fields": {"userId": 1, "studyUuid": 1}, "options": {}} +{"collections": ["audit"], "fields": {"action": 1, "studyUuid": 1}, "options": {}} +{"collections": ["audit"], "fields": {"resource": 1, "studyUuid": 1}, "options": {}} +{"collections": ["audit"], "fields": {"resourceId": 1, "studyUuid": 1}, "options": {}} +{"collections": ["audit"], "fields": {"resourceUuid": 1, "studyUuid": 1}, "options": {}} +{"collections": ["audit"], "fields": {"status.name": 1, "studyUuid": 1}, "options": {}} +{"collections": ["audit"], "fields": {"date": 1, "studyUuid": 1}, "options": {}} \ No newline at end of file diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManagerTest.java index 079d21acb19..bd070d707d8 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManagerTest.java @@ -132,7 +132,7 @@ public void before() throws Exception { CatalogManagerExternalResource.clearCatalog(configuration); catalogManager = new CatalogManager(configuration); - catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, false); + catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, true); fileManager = catalogManager.getFileManager(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptorTest.java index 0aeb607cc62..97f613182b3 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MetaMongoDBAdaptorTest.java @@ -18,8 +18,6 @@ import org.junit.Test; -import static org.junit.Assert.assertEquals; - /** * Created by imedina on 07/04/16. */ @@ -27,7 +25,7 @@ public class MetaMongoDBAdaptorTest extends MongoDBAdaptorTest { @Test public void createIndex() throws Exception { - catalogDBAdaptor.getCatalogMetaDBAdaptor().createIndexes(true); + catalogDBAdaptor.getCatalogMetaDBAdaptor().createIndexes(); } } \ No newline at end of file diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorTest.java index 816d0894228..98124031aec 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorTest.java @@ -165,8 +165,9 @@ Project getProject(String userId, String projectId) throws CatalogDBException { public void initDefaultCatalogDB() throws CatalogException { assertTrue(!catalogDBAdaptor.isCatalogDBReady()); - catalogDBAdaptor.installCatalogDB(configuration); - catalogDBAdaptor.getCatalogMetaDBAdaptor().createIndexes(true); + catalogDBAdaptor.createAllCollections(configuration); + catalogDBAdaptor.initialiseMetaCollection(configuration.getAdmin()); + catalogDBAdaptor.getCatalogMetaDBAdaptor().createIndexes(); // catalogDBAdaptor.initializeCatalogDB(new Admin()); /** diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java index 504b66ed12d..105db32dbfb 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java @@ -156,7 +156,11 @@ public void test() throws Exception { catalogProjectDBAdaptor.insert(new Project("p2", "project2", null, null, "Cool", null, 1, ProjectInternal.init()), user1.getId(), null); Project p2 = getProject(user1.getId(), "p2"); - catalogProjectDBAdaptor.update(p1.getUid(), new ObjectMap("internal.cellbase", new CellBaseConfiguration("url", "v")), new QueryOptions()); + catalogProjectDBAdaptor.update(p1.getUid(), new ObjectMap(ProjectDBAdaptor.QueryParams.CELLBASE.key(), + new CellBaseConfiguration("url", "v")), QueryOptions.empty()); + p1 = catalogProjectDBAdaptor.get(p1.getUid(), QueryOptions.empty()).first(); + assertEquals("url", p1.getCellbase().getUrl()); + assertEquals("v", p1.getCellbase().getVersion()); } } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java index 73558957175..75e548373c5 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java @@ -19,25 +19,22 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; +import org.bson.Document; import org.junit.rules.ExternalResource; -import org.opencb.commons.datastore.core.DataStoreServerAddress; -import org.opencb.commons.datastore.mongodb.MongoDataStore; -import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.auth.authentication.JwtManager; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.config.Configuration; -import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.LinkedList; -import java.util.List; import static org.opencb.opencga.core.common.JacksonUtils.getDefaultObjectMapper; @@ -68,7 +65,7 @@ public void before() throws Exception { Files.createDirectories(opencgaHome); configuration = Configuration.load(getClass().getResource("/configuration-test.yml").openStream()); configuration.setWorkspace(opencgaHome.resolve("sessions").toAbsolutePath().toString()); - configuration.setJobDir(opencgaHome.resolve("jobs").toAbsolutePath().toString()); + configuration.setJobDir(opencgaHome.resolve("JOBS").toAbsolutePath().toString()); clearCatalog(configuration); if (!opencgaHome.toFile().exists()) { @@ -77,7 +74,7 @@ public void before() throws Exception { } configuration.getAdmin().setSecretKey(PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH)); catalogManager = new CatalogManager(configuration); - catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, false); + catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, true); catalogManager.close(); // FIXME!! Should not need to create again the catalogManager // Have to create again the CatalogManager, as it has a random "secretKey" inside @@ -115,28 +112,34 @@ public ObjectMapper generateNewObjectMapper() { return jsonObjectMapper; } - public static void clearCatalog(Configuration configuration) throws IOException, CatalogException, URISyntaxException { - List dataStoreServerAddresses = new LinkedList<>(); - for (String hostPort : configuration.getCatalog().getDatabase().getHosts()) { - if (hostPort.contains(":")) { - String[] split = hostPort.split(":"); - Integer port = Integer.valueOf(split[1]); - dataStoreServerAddresses.add(new DataStoreServerAddress(split[0], port)); - } else { - dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort, 27017)); + public static void clearCatalog(Configuration configuration) throws CatalogException, URISyntaxException { + try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(configuration)) { + for (String collection : MongoDBAdaptorFactory.COLLECTIONS_LIST) { + dbAdaptorFactory.getMongoDataStore().getCollection(collection).remove(new Document(), QueryOptions.empty()); } } - MongoDataStoreManager mongoManager = new MongoDataStoreManager(dataStoreServerAddresses); -// if (catalogManager == null) { -// catalogManager = new CatalogManager(configuration); +// List dataStoreServerAddresses = new LinkedList<>(); +// for (String hostPort : configuration.getCatalog().getDatabase().getHosts()) { +// if (hostPort.contains(":")) { +// String[] split = hostPort.split(":"); +// Integer port = Integer.valueOf(split[1]); +// dataStoreServerAddresses.add(new DataStoreServerAddress(split[0], port)); +// } else { +// dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort, 27017)); +// } // } - -// MongoDataStore db = mongoManager.get(catalogConfiguration.getDatabase().getDatabase()); - MongoDataStore db = mongoManager.get(configuration.getDatabasePrefix() + "_catalog"); - db.getDb().drop(); -// mongoManager.close(catalogConfiguration.getDatabase().getDatabase()); - mongoManager.close(configuration.getDatabasePrefix() + "_catalog"); +// MongoDataStoreManager mongoManager = new MongoDataStoreManager(dataStoreServerAddresses); +// +//// if (catalogManager == null) { +//// catalogManager = new CatalogManager(configuration); +//// } +// +//// MongoDataStore db = mongoManager.get(catalogConfiguration.getDatabase().getDatabase()); +// MongoDataStore db = mongoManager.get(configuration.getDatabasePrefix() + "_catalog"); +// db.getDb().drop(); +//// mongoManager.close(catalogConfiguration.getDatabase().getDatabase()); +// mongoManager.close(configuration.getDatabasePrefix() + "_catalog"); Path rootdir = Paths.get(UriUtils.createDirectoryUri(configuration.getWorkspace())); deleteFolderTree(rootdir.toFile()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java index 17adbf25626..5140bf7e27d 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java @@ -569,11 +569,11 @@ public void createRepeatedInterpretationPrimaryFindings() throws CatalogExceptio VariantAvro variantAvro = new VariantAvro("id1", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); ClinicalVariantEvidence evidence = new ClinicalVariantEvidence().setInterpretationMethodName("method"); ClinicalVariant cv = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), - null, ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); findingList.add(cv); findingList.add(cv); variantAvro = new VariantAvro("id2", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); - cv = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), null, + cv = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); findingList.add(cv); @@ -600,11 +600,11 @@ public void createRepeatedInterpretationSecondaryFindings() throws CatalogExcept VariantAvro variantAvro = new VariantAvro("id1", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); ClinicalVariantEvidence evidence = new ClinicalVariantEvidence().setInterpretationMethodName("method"); ClinicalVariant cv = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), - null, ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); findingList.add(cv); findingList.add(cv); variantAvro = new VariantAvro("id2", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); - cv = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), null, + cv = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); findingList.add(cv); @@ -631,11 +631,11 @@ public void updatePrimaryFindings() throws CatalogException { VariantAvro variantAvro = new VariantAvro("id1", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); ClinicalVariantEvidence evidence = new ClinicalVariantEvidence().setInterpretationMethodName("method"); ClinicalVariant cv1 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), - null, ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); findingList.add(cv1); variantAvro = new VariantAvro("id2", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); ClinicalVariant cv2 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), - null, ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); findingList.add(cv2); ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis() @@ -658,7 +658,7 @@ public void updatePrimaryFindings() throws CatalogException { variantAvro = new VariantAvro("id3", null, "chr3", 2, 3, "", "", "+", null, 1, null, null, null); evidence = new ClinicalVariantEvidence().setInterpretationMethodName("method2"); ClinicalVariant cv3 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), - null, ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); findingList.add(cv3); InterpretationUpdateParams updateParams = new InterpretationUpdateParams() @@ -753,7 +753,7 @@ public void updatePrimaryFindings() throws CatalogException { // Remove finding with missing id variantAvro = new VariantAvro("", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); evidence = new ClinicalVariantEvidence().setInterpretationMethodName("method"); - cv1 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), null, + cv1 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); updateParams = new InterpretationUpdateParams() @@ -793,11 +793,11 @@ public void updateSecondaryFindings() throws CatalogException { VariantAvro variantAvro = new VariantAvro("id1", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); ClinicalVariantEvidence evidence = new ClinicalVariantEvidence().setInterpretationMethodName("method"); ClinicalVariant cv1 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), - null, ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); findingList.add(cv1); variantAvro = new VariantAvro("id2", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); ClinicalVariant cv2 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), - null, ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); findingList.add(cv2); ClinicalAnalysis ca = new ClinicalAnalysis() @@ -822,7 +822,7 @@ public void updateSecondaryFindings() throws CatalogException { variantAvro = new VariantAvro("id3", null, "chr3", 2, 3, "", "", "+", null, 1, null, null, null); evidence = new ClinicalVariantEvidence().setInterpretationMethodName("method2"); ClinicalVariant cv3 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), - null, ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); + ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); findingList.add(cv3); InterpretationUpdateParams updateParams = new InterpretationUpdateParams() @@ -920,7 +920,7 @@ public void updateSecondaryFindings() throws CatalogException { // Remove finding with missing id variantAvro = new VariantAvro("", null, "chr2", 1, 2, "", "", "+", null, 1, null, null, null); evidence = new ClinicalVariantEvidence().setInterpretationMethodName("method"); - cv1 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), null, + cv1 = new ClinicalVariant(variantAvro, Collections.singletonList(evidence), null, null, new ClinicalDiscussion(), ClinicalVariant.Status.NOT_REVIEWED, Collections.emptyList(), null); updateParams = new InterpretationUpdateParams() @@ -2453,7 +2453,7 @@ public void checkFamilyMembersOrder() throws CatalogException { @Test public void createClinicalAnalysisWithPanels() throws CatalogException { - catalogManager.getPanelManager().importFromSource(STUDY, "cancer-gene-census", "", sessionIdUser); + catalogManager.getPanelManager().importFromSource(STUDY, "gene-census", "", sessionIdUser); Panel panel = catalogManager.getPanelManager().search(STUDY, new Query(), QueryOptions.empty(), sessionIdUser).first(); DataResult dummyFamily = createDummyFamily(); @@ -2489,7 +2489,7 @@ public void createClinicalAnalysisWithPanels() throws CatalogException { @Test public void createInterpretationWithPanels() throws CatalogException { - catalogManager.getPanelManager().importFromSource(STUDY, "cancer-gene-census", "", sessionIdUser); + catalogManager.getPanelManager().importFromSource(STUDY, "gene-census", "", sessionIdUser); Panel panel = catalogManager.getPanelManager().search(STUDY, new Query(), QueryOptions.empty(), sessionIdUser).first(); ClinicalAnalysis ca = createDummyEnvironment(true, false).first(); @@ -2508,7 +2508,7 @@ public void createInterpretationWithPanels() throws CatalogException { @Test public void updatePanelsInClinicalAnalysis() throws CatalogException { - catalogManager.getPanelManager().importFromSource(STUDY, "cancer-gene-census", "", sessionIdUser); + catalogManager.getPanelManager().importFromSource(STUDY, "gene-census", "", sessionIdUser); Panel panel = catalogManager.getPanelManager().search(STUDY, new Query(), QueryOptions.empty(), sessionIdUser).first(); DataResult dummyFamily = createDummyFamily(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java index 6e561d33e22..7cacbd11d8c 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java @@ -321,7 +321,7 @@ public void createComplexFamily() throws CatalogException { Map pGrandfather = roles.get("p_grandfather"); assertEquals(5, pGrandfather.size()); - assertEquals(Family.FamiliarRelationship.SON, pGrandfather.get("father")); + assertEquals(Family.FamiliarRelationship.CHILD_OF_UNKNOWN_SEX, pGrandfather.get("father")); assertEquals(Family.FamiliarRelationship.GRANDCHILD, pGrandfather.get("proband")); assertEquals(Family.FamiliarRelationship.GRANDCHILD, pGrandfather.get("sibling")); assertEquals(Family.FamiliarRelationship.GRANDSON, pGrandfather.get("brother")); @@ -329,7 +329,7 @@ public void createComplexFamily() throws CatalogException { Map pGrandmother = roles.get("p_grandmother"); assertEquals(5, pGrandmother.size()); - assertEquals(Family.FamiliarRelationship.SON, pGrandmother.get("father")); + assertEquals(Family.FamiliarRelationship.CHILD_OF_UNKNOWN_SEX, pGrandmother.get("father")); assertEquals(Family.FamiliarRelationship.GRANDCHILD, pGrandmother.get("proband")); assertEquals(Family.FamiliarRelationship.GRANDCHILD, pGrandmother.get("sibling")); assertEquals(Family.FamiliarRelationship.GRANDSON, pGrandmother.get("brother")); @@ -337,7 +337,7 @@ public void createComplexFamily() throws CatalogException { Map mGrandfather = roles.get("m_grandfather"); assertEquals(5, mGrandfather.size()); - assertEquals(Family.FamiliarRelationship.DAUGHTER, mGrandfather.get("mother")); + assertEquals(Family.FamiliarRelationship.CHILD_OF_UNKNOWN_SEX, mGrandfather.get("mother")); assertEquals(Family.FamiliarRelationship.GRANDCHILD, mGrandfather.get("proband")); assertEquals(Family.FamiliarRelationship.GRANDCHILD, mGrandfather.get("sibling")); assertEquals(Family.FamiliarRelationship.GRANDSON, mGrandfather.get("brother")); @@ -345,7 +345,7 @@ public void createComplexFamily() throws CatalogException { Map mGrandmother = roles.get("m_grandmother"); assertEquals(5, mGrandmother.size()); - assertEquals(Family.FamiliarRelationship.DAUGHTER, mGrandmother.get("mother")); + assertEquals(Family.FamiliarRelationship.CHILD_OF_UNKNOWN_SEX, mGrandmother.get("mother")); assertEquals(Family.FamiliarRelationship.GRANDCHILD, mGrandmother.get("proband")); assertEquals(Family.FamiliarRelationship.GRANDCHILD, mGrandmother.get("sibling")); assertEquals(Family.FamiliarRelationship.GRANDSON, mGrandmother.get("brother")); @@ -1050,9 +1050,10 @@ public void createFamilyMissingMember() throws CatalogException { Family family = new Family("Martinez-Martinez", "Martinez-Martinez", Arrays.asList(phenotype1, phenotype2), null, Arrays.asList(relFather, relChild1, relChild2), "", 3, Collections.emptyList(), Collections.emptyMap()); - thrown.expect(CatalogException.class); - thrown.expectMessage("not present in the members list"); - familyManager.create(STUDY, family, QueryOptions.empty(), sessionIdUser); + family = familyManager.create(STUDY, family, new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionIdUser).first(); + assertEquals(3, family.getMembers().size()); + assertTrue(Arrays.asList(relFather.getId(), relChild2.getId(), relChild1.getId()) + .containsAll(family.getMembers().stream().map(Individual::getId).collect(Collectors.toList()))); } @Test @@ -1182,9 +1183,12 @@ public void updateFamilyMissingMember() throws CatalogException { new IndividualReferenceParam().setId("child3"), new IndividualReferenceParam().setId("father"))); - thrown.expect(CatalogException.class); - thrown.expectMessage("not present in the members list"); - familyManager.update(STUDY, originalFamily.first().getId(), updateParams, QueryOptions.empty(), sessionIdUser); + Family family = familyManager.update(STUDY, originalFamily.first().getId(), updateParams, + new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), sessionIdUser).first(); + assertEquals(2, family.getMembers().size()); + assertEquals(2, family.getRoles().size()); + assertTrue(Arrays.asList("child3", "father") + .containsAll(family.getMembers().stream().map(Individual::getId).collect(Collectors.toList()))); } @Test diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java index cd0f2b7fd8c..f84bf7a400e 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java @@ -293,19 +293,7 @@ public void createWithBase64FileWrongPath1Test() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("type"); thrown.expectMessage("path"); - fileManager.create(studyFqn, params, true, token).first(); - } - - @Test - public void createWithBase64FileWrongPath2Test() throws CatalogException { - String base64 = "iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAApgAAAKYB3X3/OAAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAAANCSURBVEiJtZZPbBtFFMZ/M7ubXdtdb1xSFyeilBapySVU8h8OoFaooFSqiihIVIpQBKci6KEg9Q6H9kovIHoCIVQJJCKE1ENFjnAgcaSGC6rEnxBwA04Tx43t2FnvDAfjkNibxgHxnWb2e/u992bee7tCa00YFsffekFY+nUzFtjW0LrvjRXrCDIAaPLlW0nHL0SsZtVoaF98mLrx3pdhOqLtYPHChahZcYYO7KvPFxvRl5XPp1sN3adWiD1ZAqD6XYK1b/dvE5IWryTt2udLFedwc1+9kLp+vbbpoDh+6TklxBeAi9TL0taeWpdmZzQDry0AcO+jQ12RyohqqoYoo8RDwJrU+qXkjWtfi8Xxt58BdQuwQs9qC/afLwCw8tnQbqYAPsgxE1S6F3EAIXux2oQFKm0ihMsOF71dHYx+f3NND68ghCu1YIoePPQN1pGRABkJ6Bus96CutRZMydTl+TvuiRW1m3n0eDl0vRPcEysqdXn+jsQPsrHMquGeXEaY4Yk4wxWcY5V/9scqOMOVUFthatyTy8QyqwZ+kDURKoMWxNKr2EeqVKcTNOajqKoBgOE28U4tdQl5p5bwCw7BWquaZSzAPlwjlithJtp3pTImSqQRrb2Z8PHGigD4RZuNX6JYj6wj7O4TFLbCO/Mn/m8R+h6rYSUb3ekokRY6f/YukArN979jcW+V/S8g0eT/N3VN3kTqWbQ428m9/8k0P/1aIhF36PccEl6EhOcAUCrXKZXXWS3XKd2vc/TRBG9O5ELC17MmWubD2nKhUKZa26Ba2+D3P+4/MNCFwg59oWVeYhkzgN/JDR8deKBoD7Y+ljEjGZ0sosXVTvbc6RHirr2reNy1OXd6pJsQ+gqjk8VWFYmHrwBzW/n+uMPFiRwHB2I7ih8ciHFxIkd/3Omk5tCDV1t+2nNu5sxxpDFNx+huNhVT3/zMDz8usXC3ddaHBj1GHj/As08fwTS7Kt1HBTmyN29vdwAw+/wbwLVOJ3uAD1wi/dUH7Qei66PfyuRj4Ik9is+hglfbkbfR3cnZm7chlUWLdwmprtCohX4HUtlOcQjLYCu+fzGJH2QRKvP3UNz8bWk1qMxjGTOMThZ3kvgLI5AzFfo379UAAAAASUVORK5CYII="; - FileCreateParams params = new FileCreateParams() - .setContent(base64) - .setType(File.Type.FILE) - .setPath("/files/folder"); - thrown.expect(CatalogException.class); - thrown.expectMessage("path"); - fileManager.create(studyFqn, params, true, token).first(); + fileManager.create(studyFqn, params, true, token); } @Test diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java index 97e2688e15f..d65bfeeb6ec 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/PanelManagerTest.java @@ -94,7 +94,7 @@ public void createTest() throws IOException, CatalogException { @Test public void importFromSource() throws CatalogException { - OpenCGAResult cancer = panelManager.importFromSource(studyFqn, "cancer-gene-census", null, sessionIdUser); + OpenCGAResult cancer = panelManager.importFromSource(studyFqn, "gene-census", null, sessionIdUser); assertEquals(1, cancer.getNumInserted()); OpenCGAResult panelApp = panelManager.importFromSource(studyFqn, "panelapp", "Thoracic_aortic_aneurysm_and_dissection-PanelAppId-700,VACTERL-like_phenotypes-PanelAppId-101", sessionIdUser); @@ -105,19 +105,19 @@ public void importFromSource() throws CatalogException { public void importFromSourceInvalidId() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("Unknown panel"); - panelManager.importFromSource(studyFqn, "cancer-gene-census", "ZSR222", sessionIdUser); + panelManager.importFromSource(studyFqn, "gene-census", "ZSR222", sessionIdUser); } @Test public void importFromInvalidSource() throws CatalogException { thrown.expect(CatalogException.class); thrown.expectMessage("Unknown source"); - panelManager.importFromSource(studyFqn, "cancer-gene-census-wrong", null, sessionIdUser); + panelManager.importFromSource(studyFqn, "gene-census-wrong", null, sessionIdUser); } @Test public void updateTest() throws CatalogException { - panelManager.importFromSource(studyFqn, "cancer-gene-census", null, sessionIdUser); + panelManager.importFromSource(studyFqn, "gene-census", null, sessionIdUser); Panel panel = panelManager.get(studyFqn, "gene-census", QueryOptions.empty(), sessionIdUser).first(); assertEquals(1, panel.getVersion()); assertEquals((int) panel.getStats().get("numberOfRegions"), panel.getVariants().size()); @@ -168,7 +168,7 @@ public void updateTest() throws CatalogException { @Test public void deletePanelTest() throws CatalogException { - panelManager.importFromSource(studyFqn, "cancer-gene-census", null, sessionIdUser); + panelManager.importFromSource(studyFqn, "gene-census", null, sessionIdUser); Panel panel = panelManager.get(studyFqn, "gene-census", QueryOptions.empty(), sessionIdUser).first(); assertEquals(1, panel.getVersion()); @@ -186,7 +186,7 @@ public void deletePanelTest() throws CatalogException { @Test public void deletePanelWithVersionsTest() throws CatalogException { - panelManager.importFromSource(studyFqn, "cancer-gene-census", null, sessionIdUser); + panelManager.importFromSource(studyFqn, "gene-census", null, sessionIdUser); Panel panel = panelManager.get(studyFqn, "gene-census", QueryOptions.empty(), sessionIdUser).first(); assertEquals(1, panel.getVersion()); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java index b5428c851cb..d2fee6fa1e7 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java @@ -201,7 +201,7 @@ public void updateCellbaseInProject() throws CatalogException, JsonProcessingExc null, "GRCh38", INCLUDE_RESULT, sessionIdUser).first(); assertNotNull(pr.getCellbase()); assertEquals("https://ws.zettagenomics.com/cellbase", pr.getCellbase().getUrl()); - assertEquals("v5", pr.getCellbase().getVersion()); + assertEquals(ParamConstants.CELLBASE_VERSION, pr.getCellbase().getVersion()); CellBaseConfiguration cb = new CellBaseConfiguration("https://ws.opencb.org/cellbase", "v3"); OpenCGAResult update = catalogManager.getProjectManager().setCellbaseConfiguration(pr.getId(), diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java index 703babf28cf..39562c9d103 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java @@ -440,7 +440,7 @@ public void updateQualityControlTest1() throws CatalogException { sampleVariantStats.setTiTvRatio((float) 15.2); sampleQcVariantStats.add(new SampleQcVariantStats("v2", "", null, sampleVariantStats)); - SampleVariantQualityControlMetrics metrics = new SampleVariantQualityControlMetrics(sampleQcVariantStats, null, null, null); + SampleVariantQualityControlMetrics metrics = new SampleVariantQualityControlMetrics(sampleQcVariantStats, null, null, null, null); SampleQualityControl qualityControl = new SampleQualityControl(null, null, metrics); OpenCGAResult result = catalogManager.getSampleManager().update(studyFqn, "sample", @@ -483,7 +483,7 @@ public void updateQualityControlTest1() throws CatalogException { sampleVariantStats.setVariantCount(15); sampleVariantStats.setTiTvRatio((float) 3.5); sampleQcVariantStats.add(new SampleQcVariantStats("v1", "", null, sampleVariantStats)); - metrics = new SampleVariantQualityControlMetrics(sampleQcVariantStats, null, null, null); + metrics = new SampleVariantQualityControlMetrics(sampleQcVariantStats, null, null, null, null); qualityControl = new SampleQualityControl(null, null, metrics); // And update sample @@ -614,7 +614,7 @@ public void updateQualityControlTest2() throws CatalogException { sampleVariantStats.setTiTvRatio((float) 15.2); sampleQcVariantStats.add(new SampleQcVariantStats("v2", "", null, sampleVariantStats)); - SampleVariantQualityControlMetrics metrics = new SampleVariantQualityControlMetrics(sampleQcVariantStats, null, null, null); + SampleVariantQualityControlMetrics metrics = new SampleVariantQualityControlMetrics(sampleQcVariantStats, null, null, null, null); SampleQualityControl qualityControl = new SampleQualityControl(null, null, metrics); OpenCGAResult result = catalogManager.getSampleManager().update(studyFqn, "sample", @@ -648,7 +648,7 @@ public void updateQualityControlTest2() throws CatalogException { sampleVariantStats.setVariantCount(15); sampleVariantStats.setTiTvRatio((float) 3.5); sampleQcVariantStats.add(new SampleQcVariantStats("v1", "", null, sampleVariantStats)); - metrics = new SampleVariantQualityControlMetrics(sampleQcVariantStats, null, null, null); + metrics = new SampleVariantQualityControlMetrics(sampleQcVariantStats, null, null, null, null); qualityControl = new SampleQualityControl(null, null, metrics); // And update sample @@ -868,14 +868,14 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { ClinicalAnalysis clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), token).first(); assertEquals(2, clinicalResult.getProband().getVersion()); - assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version + assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(1, clinicalResult.getFamily().getVersion()); assertEquals(2, clinicalResult.getFamily().getMembers().get(0).getVersion()); // proband version assertEquals(2, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), token).first(); assertEquals(3, clinicalResult.getProband().getVersion()); - assertEquals(3, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version + assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(2, clinicalResult.getFamily().getVersion()); assertEquals(3, clinicalResult.getFamily().getMembers().get(0).getVersion()); // proband version assertEquals(2, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version @@ -901,7 +901,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), token).first(); assertEquals(2, clinicalResult.getProband().getVersion()); - assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version + assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(1, clinicalResult.getFamily().getVersion()); assertEquals(2, clinicalResult.getFamily().getMembers().get(0).getVersion()); // proband version assertEquals(2, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version @@ -909,7 +909,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical2", QueryOptions.empty(), token).first(); assertEquals(3, clinicalResult.getProband().getVersion()); - assertEquals(3, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version + assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(3, clinicalResult.getFamily().getVersion()); assertEquals(3, clinicalResult.getFamily().getMembers().get(0).getVersion()); // proband version assertEquals(3, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version @@ -940,7 +940,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { clinicalResult = catalogManager.getClinicalAnalysisManager().get(studyFqn, "clinical", QueryOptions.empty(), token).first(); assertEquals(2, clinicalResult.getProband().getVersion()); - assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample1 version + assertEquals(2, clinicalResult.getProband().getSamples().get(0).getVersion()); // sample2 version assertEquals(1, clinicalResult.getFamily().getVersion()); assertEquals(2, clinicalResult.getFamily().getMembers().get(0).getVersion()); // proband version assertEquals(2, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version @@ -952,6 +952,7 @@ public void testUpdateWithLockedClinicalAnalysis() throws CatalogException { assertEquals(5, clinicalResult.getFamily().getVersion()); assertEquals(4, clinicalResult.getFamily().getMembers().get(0).getVersion()); // proband version assertEquals(4, clinicalResult.getFamily().getMembers().get(1).getVersion()); // father version + assertEquals(4, clinicalResult.getFamily().getMembers().get(1).getSamples().get(0).getVersion()); // sample3 version assertEquals(3, clinicalResult.getFamily().getMembers().get(0).getSamples().get(0).getVersion()); // proband sample2 version } @@ -2123,6 +2124,32 @@ public void testModifySample() throws CatalogException { assertNotNull(sample.getAttributes()); } + @Test + public void associateSameSampleAndIndividualInDifferentStudies() throws CatalogException { + String id = "myUniqueId"; + for (String studyId : Arrays.asList("st1", "st2")) { + System.out.println("Study " + studyId); + catalogManager.getStudyManager().create(project1, new Study().setId(studyId), QueryOptions.empty(), token); + + catalogManager.getSampleManager().create(studyId, new Sample().setId(id), QueryOptions.empty(), token); + catalogManager.getIndividualManager().create(studyId, new Individual().setId(id), QueryOptions.empty(), token); + catalogManager.getSampleManager().update(studyId, id, new SampleUpdateParams().setIndividualId(id), QueryOptions.empty(), token); + + OpenCGAResult sampleResult = catalogManager.getSampleManager().get(studyId, id, QueryOptions.empty(), token); + assertEquals(1, sampleResult.getNumResults()); + assertEquals(id, sampleResult.first().getIndividualId()); + assertEquals(2, sampleResult.first().getVersion()); + + OpenCGAResult individualResult = catalogManager.getIndividualManager().get(studyId, id, QueryOptions.empty(), token); + assertEquals(1, individualResult.getNumResults()); + assertEquals(2, individualResult.first().getVersion()); + assertEquals(1, individualResult.first().getSamples().size()); + assertEquals(id, individualResult.first().getSamples().get(0).getId()); + assertEquals(2, individualResult.first().getSamples().get(0).getVersion()); + } + + } + @Test public void testGetSampleAndIndividualWithPermissionsChecked() throws CatalogException { String sampleId1 = catalogManager.getSampleManager() @@ -2351,7 +2378,7 @@ public void checkAnonymousUserWithNoPermissions() throws CatalogException { ParamUtils.AclAction.ADD, this.token); thrown.expect(CatalogException.class); - thrown.expectMessage("view study"); + thrown.expectMessage("view any study"); catalogManager.getStudyManager().get(studyFqn, QueryOptions.empty(), ""); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogFileUtilsTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogFileUtilsTest.java index 10d3ea891fa..b1e7f4b565a 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogFileUtilsTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogFileUtilsTest.java @@ -72,7 +72,7 @@ public void before() throws CatalogException, IOException, URISyntaxException { CatalogManagerExternalResource.clearCatalog(configuration); catalogManager = new CatalogManager(configuration); - catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, false); + catalogManager.installCatalogDB(configuration.getAdmin().getSecretKey(), TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, true); //Create USER catalogManager.getUserManager().create("user", "name", "mi@mail.com", TestParamConstants.PASSWORD, "", null, Account.AccountType.FULL, null); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoaderTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoaderTest.java index fc700751e50..032f8c82d57 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoaderTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/CatalogSampleAnnotationsLoaderTest.java @@ -16,10 +16,7 @@ package org.opencb.opencga.catalog.utils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.*; import org.opencb.biodata.models.pedigree.Individual; import org.opencb.biodata.models.pedigree.Pedigree; import org.opencb.commons.datastore.core.DataResult; @@ -48,6 +45,7 @@ import java.net.URL; import java.util.*; +@Ignore public class CatalogSampleAnnotationsLoaderTest extends GenericTest { private static final List populations = Arrays.asList("ACB", "ASW", "BEB", "CDX", "CEU", "CHB", "CHS", "CLM", "ESN", "FIN", @@ -70,7 +68,7 @@ public static void beforeClass() throws IOException, CatalogException, URISyntax catalogManager.deleteCatalogDB(TestParamConstants.ADMIN_PASSWORD); } catalogManager.installCatalogDB(PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH), - TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, false); + TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", "", true, true); loader = new CatalogSampleAnnotationsLoader(catalogManager); diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index 2319aedb11f..0d71ca7639a 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -7,7 +7,7 @@ openRegister: true databasePrefix: "opencga_test" workspace: "/tmp/opencga/sessions" -jobDir: "/tmp/opencga/jobs" +jobDir: "/tmp/opencga/JOBS" admin: secretKey: "asidnadh19rh230qncfascd1.rffzasf.asd.ad.12ddeASDAsd12e1d.adsx" diff --git a/opencga-client/pom.xml b/opencga-client/pom.xml index ff7d1cd1d36..50e45d40b24 100644 --- a/opencga-client/pom.xml +++ b/opencga-client/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-client/src/main/R/R/Admin-methods.R b/opencga-client/src/main/R/R/Admin-methods.R index fd6c095f49c..98c091d710f 100644 --- a/opencga-client/src/main/R/R/Admin-methods.R +++ b/opencga-client/src/main/R/R/Admin-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Alignment-methods.R b/opencga-client/src/main/R/R/Alignment-methods.R index 98f49ccbfd6..0ee9890c02e 100644 --- a/opencga-client/src/main/R/R/Alignment-methods.R +++ b/opencga-client/src/main/R/R/Alignment-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -165,13 +165,13 @@ setMethod("alignmentClient", "OpencgaR", function(OpencgaR, endpointName, params subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), #' @section Endpoint /{apiVersion}/analysis/alignment/qc/run: - #' Compute quality control (QC) metrics for a given alignment file (including samtools stats, samtools flag stats, FastQC and HS metrics). + #' Compute quality control (QC) metrics for a given alignment file: samtools stats, samtools flag stats and FastQC metrics. #' @param study study. #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobDescription Job description. #' @param jobTags Job tags. - #' @param data Alignment quality control (QC) parameters. It computes: stats, flag stats, fastqc and hybrid-selection metrics. The BAM file is mandatory ever but the BED fileand the dictionary files are only mandatory for computing hybrid-selection (HS) metrics. In order to skip some metrics, use the following keywords (separated by commas): stats, flagstats, fastqc and hsmetrics. + #' @param data Alignment quality control (QC) parameters. It computes: stats, flag stats and fastqc metrics. The BAM file ID is mandatory and in order to skip some metrics, use the following keywords (separated by commas): stats, flagstats, fastqc. runQc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="alignment/qc", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/R/R/AllGenerics.R b/opencga-client/src/main/R/R/AllGenerics.R index 29e94ee0aa5..0a574313344 100644 --- a/opencga-client/src/main/R/R/AllGenerics.R +++ b/opencga-client/src/main/R/R/AllGenerics.R @@ -1,21 +1,21 @@ # ############################################################################## ## UserClient -setGeneric("userClient", function(OpencgaR, user, filterId, users, endpointName, params=NULL, ...) +setGeneric("userClient", function(OpencgaR, filterId, users, user, endpointName, params=NULL, ...) standardGeneric("userClient")) # ############################################################################## ## ProjectClient -setGeneric("projectClient", function(OpencgaR, projects, project, endpointName, params=NULL, ...) +setGeneric("projectClient", function(OpencgaR, project, projects, endpointName, params=NULL, ...) standardGeneric("projectClient")) # ############################################################################## ## StudyClient -setGeneric("studyClient", function(OpencgaR, study, members, variableSet, templateId, studies, group, endpointName, params=NULL, ...) +setGeneric("studyClient", function(OpencgaR, variableSet, members, group, studies, templateId, study, endpointName, params=NULL, ...) standardGeneric("studyClient")) # ############################################################################## ## FileClient -setGeneric("fileClient", function(OpencgaR, members, file, annotationSet, folder, files, endpointName, params=NULL, ...) +setGeneric("fileClient", function(OpencgaR, file, members, files, annotationSet, folder, endpointName, params=NULL, ...) standardGeneric("fileClient")) # ############################################################################## @@ -25,22 +25,22 @@ setGeneric("jobClient", function(OpencgaR, members, jobs, job, endpointName, par # ############################################################################## ## SampleClient -setGeneric("sampleClient", function(OpencgaR, annotationSet, samples, members, sample, endpointName, params=NULL, ...) +setGeneric("sampleClient", function(OpencgaR, annotationSet, sample, members, samples, endpointName, params=NULL, ...) standardGeneric("sampleClient")) # ############################################################################## ## IndividualClient -setGeneric("individualClient", function(OpencgaR, individuals, individual, members, annotationSet, endpointName, params=NULL, ...) +setGeneric("individualClient", function(OpencgaR, annotationSet, individuals, members, individual, endpointName, params=NULL, ...) standardGeneric("individualClient")) # ############################################################################## ## FamilyClient -setGeneric("familyClient", function(OpencgaR, annotationSet, families, members, family, endpointName, params=NULL, ...) +setGeneric("familyClient", function(OpencgaR, annotationSet, members, family, families, endpointName, params=NULL, ...) standardGeneric("familyClient")) # ############################################################################## ## CohortClient -setGeneric("cohortClient", function(OpencgaR, cohorts, cohort, members, annotationSet, endpointName, params=NULL, ...) +setGeneric("cohortClient", function(OpencgaR, annotationSet, cohorts, members, cohort, endpointName, params=NULL, ...) standardGeneric("cohortClient")) # ############################################################################## @@ -60,7 +60,7 @@ setGeneric("variantClient", function(OpencgaR, endpointName, params=NULL, ...) # ############################################################################## ## ClinicalClient -setGeneric("clinicalClient", function(OpencgaR, members, interpretations, clinicalAnalyses, interpretation, clinicalAnalysis, endpointName, params=NULL, ...) +setGeneric("clinicalClient", function(OpencgaR, members, clinicalAnalyses, interpretations, interpretation, clinicalAnalysis, endpointName, params=NULL, ...) standardGeneric("clinicalClient")) # ############################################################################## @@ -75,7 +75,7 @@ setGeneric("metaClient", function(OpencgaR, endpointName, params=NULL, ...) # ############################################################################## ## GA4GHClient -setGeneric("ga4ghClient", function(OpencgaR, study, file, endpointName, params=NULL, ...) +setGeneric("ga4ghClient", function(OpencgaR, file, study, endpointName, params=NULL, ...) standardGeneric("ga4ghClient")) # ############################################################################## diff --git a/opencga-client/src/main/R/R/Clinical-methods.R b/opencga-client/src/main/R/R/Clinical-methods.R index 0d6fc4d1495..adae227c5be 100644 --- a/opencga-client/src/main/R/R/Clinical-methods.R +++ b/opencga-client/src/main/R/R/Clinical-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -58,7 +58,7 @@ #' [*]: Required parameter #' @export -setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, interpretations, clinicalAnalyses, interpretation, clinicalAnalysis, endpointName, params=NULL, ...) { +setMethod("clinicalClient", "OpencgaR", function(OpencgaR, members, clinicalAnalyses, interpretations, interpretation, clinicalAnalysis, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/analysis/clinical/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Cohort-methods.R b/opencga-client/src/main/R/R/Cohort-methods.R index b6e849cc7c1..50e51d468ca 100644 --- a/opencga-client/src/main/R/R/Cohort-methods.R +++ b/opencga-client/src/main/R/R/Cohort-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("cohortClient", "OpencgaR", function(OpencgaR, cohorts, cohort, members, annotationSet, endpointName, params=NULL, ...) { +setMethod("cohortClient", "OpencgaR", function(OpencgaR, annotationSet, cohorts, members, cohort, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/cohorts/acl/{members}/update: @@ -100,7 +100,7 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, cohorts, cohort, member #' @param id Comma separated list of cohort IDs up to a maximum of 100. #' @param name Comma separated list of cohort names up to a maximum of 100. #' @param uuid Comma separated list of cohort IDs up to a maximum of 100. - #' @param type Cohort type. Allowed values: ['CASE_CONTROL CASE_SET CONTROL_SET PAIRED PAIRED_TUMOR AGGREGATE TIME_SERIES FAMILY TRIO COLLECTION'] + #' @param type Cohort type. #' @param creationDate creationDate. #' @param modificationDate modificationDate. #' @param deleted deleted. @@ -150,7 +150,7 @@ setMethod("cohortClient", "OpencgaR", function(OpencgaR, cohorts, cohort, member #' @param id Comma separated list of cohort IDs up to a maximum of 100. #' @param name Comma separated list of cohort names up to a maximum of 100. #' @param uuid Comma separated list of cohort IDs up to a maximum of 100. - #' @param type Cohort type. Allowed values: ['CASE_CONTROL CASE_SET CONTROL_SET PAIRED PAIRED_TUMOR AGGREGATE TIME_SERIES FAMILY TRIO COLLECTION'] + #' @param type Cohort type. #' @param creationDate creationDate. #' @param modificationDate modificationDate. #' @param deleted deleted. diff --git a/opencga-client/src/main/R/R/Family-methods.R b/opencga-client/src/main/R/R/Family-methods.R index f52ecf3d6f7..8a70a24a16a 100644 --- a/opencga-client/src/main/R/R/Family-methods.R +++ b/opencga-client/src/main/R/R/Family-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +38,7 @@ #' [*]: Required parameter #' @export -setMethod("familyClient", "OpencgaR", function(OpencgaR, annotationSet, families, members, family, endpointName, params=NULL, ...) { +setMethod("familyClient", "OpencgaR", function(OpencgaR, annotationSet, members, family, families, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/families/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/File-methods.R b/opencga-client/src/main/R/R/File-methods.R index 73f4a25f61a..73bf0fdb3e8 100644 --- a/opencga-client/src/main/R/R/File-methods.R +++ b/opencga-client/src/main/R/R/File-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -54,7 +54,7 @@ #' [*]: Required parameter #' @export -setMethod("fileClient", "OpencgaR", function(OpencgaR, members, file, annotationSet, folder, files, endpointName, params=NULL, ...) { +setMethod("fileClient", "OpencgaR", function(OpencgaR, file, members, files, annotationSet, folder, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/files/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/GA4GH-methods.R b/opencga-client/src/main/R/R/GA4GH-methods.R index f45a836b7e5..e61ac622cd5 100644 --- a/opencga-client/src/main/R/R/GA4GH-methods.R +++ b/opencga-client/src/main/R/R/GA4GH-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -31,7 +31,7 @@ #' [*]: Required parameter #' @export -setMethod("ga4ghClient", "OpencgaR", function(OpencgaR, study, file, endpointName, params=NULL, ...) { +setMethod("ga4ghClient", "OpencgaR", function(OpencgaR, file, study, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/ga4gh/reads/search: diff --git a/opencga-client/src/main/R/R/Individual-methods.R b/opencga-client/src/main/R/R/Individual-methods.R index 17602856753..da6f4c5a631 100644 --- a/opencga-client/src/main/R/R/Individual-methods.R +++ b/opencga-client/src/main/R/R/Individual-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("individualClient", "OpencgaR", function(OpencgaR, individuals, individual, members, annotationSet, endpointName, params=NULL, ...) { +setMethod("individualClient", "OpencgaR", function(OpencgaR, annotationSet, individuals, members, individual, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/individuals/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Job-methods.R b/opencga-client/src/main/R/R/Job-methods.R index dd1f2d91848..17f58fb03bf 100644 --- a/opencga-client/src/main/R/R/Job-methods.R +++ b/opencga-client/src/main/R/R/Job-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Meta-methods.R b/opencga-client/src/main/R/R/Meta-methods.R index b607f86b0ed..540337e8b3f 100644 --- a/opencga-client/src/main/R/R/Meta-methods.R +++ b/opencga-client/src/main/R/R/Meta-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Operation-methods.R b/opencga-client/src/main/R/R/Operation-methods.R index 80f98120c46..b86e0ec8911 100644 --- a/opencga-client/src/main/R/R/Operation-methods.R +++ b/opencga-client/src/main/R/R/Operation-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Panel-methods.R b/opencga-client/src/main/R/R/Panel-methods.R index fab3b9d1f06..626ca72350b 100644 --- a/opencga-client/src/main/R/R/Panel-methods.R +++ b/opencga-client/src/main/R/R/Panel-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Project-methods.R b/opencga-client/src/main/R/R/Project-methods.R index 9d29438ef29..e93e640ee9d 100644 --- a/opencga-client/src/main/R/R/Project-methods.R +++ b/opencga-client/src/main/R/R/Project-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -34,7 +34,7 @@ #' [*]: Required parameter #' @export -setMethod("projectClient", "OpencgaR", function(OpencgaR, projects, project, endpointName, params=NULL, ...) { +setMethod("projectClient", "OpencgaR", function(OpencgaR, project, projects, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/projects/create: diff --git a/opencga-client/src/main/R/R/Sample-methods.R b/opencga-client/src/main/R/R/Sample-methods.R index 600b407104a..5d486f4da78 100644 --- a/opencga-client/src/main/R/R/Sample-methods.R +++ b/opencga-client/src/main/R/R/Sample-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, samples, members, sample, endpointName, params=NULL, ...) { +setMethod("sampleClient", "OpencgaR", function(OpencgaR, annotationSet, sample, members, samples, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/samples/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index a1028f9a802..1a32672b875 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +46,7 @@ #' [*]: Required parameter #' @export -setMethod("studyClient", "OpencgaR", function(OpencgaR, study, members, variableSet, templateId, studies, group, endpointName, params=NULL, ...) { +setMethod("studyClient", "OpencgaR", function(OpencgaR, variableSet, members, group, studies, templateId, study, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/studies/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/User-methods.R b/opencga-client/src/main/R/R/User-methods.R index 65b4e3e0505..08925b8a812 100644 --- a/opencga-client/src/main/R/R/User-methods.R +++ b/opencga-client/src/main/R/R/User-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("userClient", "OpencgaR", function(OpencgaR, user, filterId, users, endpointName, params=NULL, ...) { +setMethod("userClient", "OpencgaR", function(OpencgaR, filterId, users, user, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/users/create: diff --git a/opencga-client/src/main/R/R/Variant-methods.R b/opencga-client/src/main/R/R/Variant-methods.R index ce55573d75d..debe0d9dbfe 100644 --- a/opencga-client/src/main/R/R/Variant-methods.R +++ b/opencga-client/src/main/R/R/Variant-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2022-12-13 18:29:34 +# Autogenerated on: 2023-01-10 10:57:29 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -35,6 +35,7 @@ #' | runGatk | /{apiVersion}/analysis/variant/gatk/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | #' | runGenomePlot | /{apiVersion}/analysis/variant/genomePlot/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | #' | runGwas | /{apiVersion}/analysis/variant/gwas/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | +#' | runHrDetect | /{apiVersion}/analysis/variant/hrDetect/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | #' | runIndex | /{apiVersion}/analysis/variant/index/run | study, jobId, jobDependsOn, jobDescription, jobTags, body[*] | #' | runIndividualQc | /{apiVersion}/analysis/variant/individual/qc/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | #' | runInferredSex | /{apiVersion}/analysis/variant/inferredSex/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | @@ -43,7 +44,7 @@ #' | runKnockout | /{apiVersion}/analysis/variant/knockout/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | #' | runMendelianError | /{apiVersion}/analysis/variant/mendelianError/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | #' | metadata | /{apiVersion}/analysis/variant/metadata | project, study, file, sample, includeStudy, includeFile, includeSample, include, exclude | -#' | queryMutationalSignature | /{apiVersion}/analysis/variant/mutationalSignature/query | study, sample, ct, biotype, fileData, filter, qual, region, gene, panel, panelModeOfInheritance, panelConfidence, panelFeatureType, panelRoleInCancer, panelIntersection, catalogues, cataloguesContent, fitMethod, nBoot, sigVersion, organ, thresholdPerc, thresholdPval, maxRareSigs, signaturesFile, rareSignaturesFile | +#' | queryMutationalSignature | /{apiVersion}/analysis/variant/mutationalSignature/query | study, sample, type, ct, biotype, fileData, filter, qual, region, gene, panel, panelModeOfInheritance, panelConfidence, panelFeatureType, panelRoleInCancer, panelIntersection, msId, msDescription | #' | runMutationalSignature | /{apiVersion}/analysis/variant/mutationalSignature/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | #' | runPlink | /{apiVersion}/analysis/variant/plink/run | study, jobId, jobDescription, jobDependsOn, jobTags, body[*] | #' | query | /{apiVersion}/analysis/variant/query | include, exclude, limit, skip, count, sort, summary, approximateCount, approximateCountSamplingSize, savedFilter, id, region, type, reference, alternate, project, study, file, filter, qual, fileData, sample, genotype, sampleData, sampleAnnotation, sampleMetadata, unknownGenotype, sampleLimit, sampleSkip, cohort, cohortStatsRef, cohortStatsAlt, cohortStatsMaf, cohortStatsMgf, cohortStatsPass, missingAlleles, missingGenotypes, score, family, familyDisorder, familySegregation, familyMembers, familyProband, includeStudy, includeFile, includeSample, includeSampleData, includeGenotype, includeSampleId, annotationExists, gene, ct, xref, biotype, proteinSubstitution, conservation, populationFrequencyAlt, populationFrequencyRef, populationFrequencyMaf, transcriptFlag, geneTraitId, go, expression, proteinKeyword, drug, functionalScore, clinical, clinicalSignificance, clinicalConfirmedStatus, customAnnotation, panel, panelModeOfInheritance, panelConfidence, panelRoleInCancer, panelFeatureType, panelIntersection, trait | @@ -262,6 +263,17 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N runGwas=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/gwas", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/analysis/variant/hrDetect/run: + #' Run HRDetect analysis for a given somatic sample. + #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. + #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + #' @param jobDescription Job description. + #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. + #' @param jobTags Job tags. + #' @param data HRDetect analysis parameters. + runHrDetect=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/hrDetect", + subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), + #' @section Endpoint /{apiVersion}/analysis/variant/index/run: #' [DEPRECATED] Use operation/variant/index. #' @param study Study [[user@]project:]study where study and project can be either the ID or UUID. @@ -358,6 +370,7 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' Run mutational signature analysis for a given sample. Use context index. #' @param study Filter variants from the given studies, these can be either the numeric ID or the alias with the format user@project:study. #' @param sample Sample name. + #' @param type Variant type. Valid values: SNV, SV. #' @param ct List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'. #' @param biotype List of biotypes, e.g. protein_coding. #' @param fileData Filter by file data (i.e. FILTER, QUAL and INFO columns from VCF file). [{file}:]{key}{op}{value}[,;]* . If no file is specified, will use all files from "file" filter. e.g. AN>200 or file_1.vcf:AN>200;file_2.vcf:AN<10 . Many fields can be combined. e.g. file_1.vcf:AN>200;DB=true;file_2.vcf:AN<10,FILTER=PASS,LowDP. @@ -371,17 +384,8 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param panelFeatureType Filter elements from specific panels by type. Accepted values : [ gene, region, str, variant ]. #' @param panelRoleInCancer Filter genes from specific panels that match certain role in cancer. Accepted values : [ both, oncogene, tumorSuppressorGene, fusion ]. #' @param panelIntersection Intersect panel genes and regions with given genes and regions from que input query. This will prevent returning variants from regions out of the panel. - #' @param catalogues File name containing mutational catalogues. Each sample catalogue is in a column, with sample names as column headers and channel. - #' @param cataloguesContent Mutational catalogues. Each sample catalogue is in a column, with sample names as column headers and channel. - #' @param fitMethod Either Fit or FitMS. If not specified then FitMS. - #' @param nBoot Number of bootstrap to be used. - #' @param sigVersion Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2. - #' @param organ When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus. - #' @param thresholdPerc Threshold in percentage of total mutations in a sample, only exposures larger than THRPERC are considered. If not specified 5. - #' @param thresholdPval P-value to determine the empirical probability that the exposure is lower than the threshold. If not specified then 0.05. - #' @param maxRareSigs Maximum number of rare signatures that are allowed to be present in each sample. If not specified 1. - #' @param signaturesFile The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures. - #' @param rareSignaturesFile The file name containing mutational signatures. Each signature is in a column, with signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures. + #' @param msId Signature ID. + #' @param msDescription Signature description. queryMutationalSignature=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/mutationalSignature", subcategoryId=NULL, action="query", params=params, httpMethod="GET", as.queryParam=NULL, ...), @@ -393,7 +397,7 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param data Mutational signature analysis params. + #' @param data Mutational signature analysis parameters to index the genome context for that sample, and to compute both catalogue counts and signature fitting. In order to skip one of them, , use the following keywords: , catalogue, fitting. runMutationalSignature=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/mutationalSignature", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), @@ -561,7 +565,7 @@ setMethod("variantClient", "OpencgaR", function(OpencgaR, endpointName, params=N #' @param jobDescription Job description. #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. #' @param jobTags Job tags. - #' @param data Sample QC analysis params. Mutational signature and genome plot are calculated for somatic samples only. + #' @param data Sample QC analysis params. Mutational signature and genome plot are calculated for somatic samples only. In order to skip some metrics, use the following keywords (separated by commas): variant-stats, signature, signature-catalogue, signature-fitting, genome-plot. runSampleQc=fetchOpenCGA(object=OpencgaR, category="analysis", categoryId=NULL, subcategory="variant/sample/qc", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java index 9146f7f064c..f56adbd5348 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -44,7 +44,7 @@ /** * This class contains methods for the Admin webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: admin */ public class AdminClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java index df0be68174c..60c8d4cb20e 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -49,7 +49,7 @@ /** * This class contains methods for the Alignment webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: analysis/alignment */ public class AlignmentClient extends AbstractParentClient { @@ -251,11 +251,9 @@ public RestResponse runPicard(PicardWrapperParams data, ObjectMap params) t } /** - * Compute quality control (QC) metrics for a given alignment file (including samtools stats, samtools flag stats, FastQC and HS - * metrics). - * @param data Alignment quality control (QC) parameters. It computes: stats, flag stats, fastqc and hybrid-selection metrics. The BAM - * file is mandatory ever but the BED fileand the dictionary files are only mandatory for computing hybrid-selection (HS) metrics. - * In order to skip some metrics, use the following keywords (separated by commas): stats, flagstats, fastqc and hsmetrics. + * Compute quality control (QC) metrics for a given alignment file: samtools stats, samtools flag stats and FastQC metrics. + * @param data Alignment quality control (QC) parameters. It computes: stats, flag stats and fastqc metrics. The BAM file ID is + * mandatory and in order to skip some metrics, use the following keywords (separated by commas): stats, flagstats, fastqc. * @param params Map containing any of the following optional parameters. * study: study. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java index b56ae0f7450..2ae1cd51edd 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,7 +51,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -60,7 +60,7 @@ /** * This class contains methods for the ClinicalAnalysis webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: analysis/clinical */ public class ClinicalAnalysisClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java index 4b58b2d197c..07fe9c2f6e9 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +46,7 @@ /** * This class contains methods for the Cohort webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: cohorts */ public class CohortClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java index 10a1586da62..82746e4cba4 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -44,7 +44,7 @@ /** * This class contains methods for the DiseasePanel webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: panels */ public class DiseasePanelClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java index 7ba1b9c19a7..b0748be26dd 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,7 @@ /** * This class contains methods for the Family webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: families */ public class FamilyClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java index 859aa8bcca7..c26c1398dbb 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,7 +43,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -52,7 +52,7 @@ /** * This class contains methods for the File webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: files */ public class FileClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java index 8317f59ad31..56b1c8e1beb 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -36,7 +36,7 @@ /** * This class contains methods for the GA4GH webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: ga4gh */ public class GA4GHClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java index 3d1fbda51ab..12c9e3c11a4 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,7 @@ /** * This class contains methods for the Individual webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: individuals */ public class IndividualClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java index b036cc853b9..508b9958860 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +46,7 @@ /** * This class contains methods for the Job webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: jobs */ public class JobClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java index b67e9a231a3..4a9d78ee9f6 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -37,7 +37,7 @@ /** * This class contains methods for the Meta webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: meta */ public class MetaClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java index 75b57f4b0a6..b31f8c15071 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,7 +32,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -41,7 +41,7 @@ /** * This class contains methods for the Project webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: projects */ public class ProjectClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java index 6736a122d4e..9a2805599f8 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,7 @@ /** * This class contains methods for the Sample webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: samples */ public class SampleClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java index 5b6551824d5..6d44741cc90 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -54,7 +54,7 @@ /** * This class contains methods for the Study webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: studies */ public class StudyClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java index 0d97bff8967..3d69c968b4c 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +46,7 @@ /** * This class contains methods for the User webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: users */ public class UserClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java index cd384f593e5..a6dc033c744 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,6 +39,7 @@ import org.opencb.opencga.core.models.variant.GatkWrapperParams; import org.opencb.opencga.core.models.variant.GenomePlotAnalysisParams; import org.opencb.opencga.core.models.variant.GwasAnalysisParams; +import org.opencb.opencga.core.models.variant.HRDetectAnalysisParams; import org.opencb.opencga.core.models.variant.IndividualQcAnalysisParams; import org.opencb.opencga.core.models.variant.InferredSexAnalysisParams; import org.opencb.opencga.core.models.variant.KnockoutAnalysisParams; @@ -61,7 +62,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -70,7 +71,7 @@ /** * This class contains methods for the Variant webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: analysis/variant */ public class VariantClient extends AbstractParentClient { @@ -378,6 +379,24 @@ public RestResponse runGwas(GwasAnalysisParams data, ObjectMap params) thro return execute("analysis", null, "variant/gwas", null, "run", params, POST, Job.class); } + /** + * Run HRDetect analysis for a given somatic sample. + * @param data HRDetect analysis parameters. + * @param params Map containing any of the following optional parameters. + * study: Study [[user@]project:]study where study and project can be either the ID or UUID. + * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + * jobDescription: Job description. + * jobDependsOn: Comma separated list of existing job IDs the job will depend on. + * jobTags: Job tags. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse runHrDetect(HRDetectAnalysisParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("analysis", null, "variant/hrDetect", null, "run", params, POST, Job.class); + } + /** * [DEPRECATED] Use operation/variant/index. * @param data Variant index params. @@ -536,6 +555,7 @@ public RestResponse metadata(ObjectMap params) throws ClientExc * study: Filter variants from the given studies, these can be either the numeric ID or the alias with the format * user@project:study. * sample: Sample name. + * type: Variant type. Valid values: SNV, SV. * ct: List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' * and 'protein_altering'. * biotype: List of biotypes, e.g. protein_coding. @@ -559,29 +579,8 @@ public RestResponse metadata(ObjectMap params) throws ClientExc * tumorSuppressorGene, fusion ]. * panelIntersection: Intersect panel genes and regions with given genes and regions from que input query. This will prevent * returning variants from regions out of the panel. - * catalogues: File name containing mutational catalogues. Each sample catalogue is in a column, with sample names as column - * headers and channel. - * cataloguesContent: Mutational catalogues. Each sample catalogue is in a column, with sample names as column headers and - * channel. - * fitMethod: Either Fit or FitMS. If not specified then FitMS. - * nBoot: Number of bootstrap to be used. - * sigVersion: Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2. - * organ: When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION is COSMICv2 or - * COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the selected - * SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, - * Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, - * Prostate, Skin, Stomach, Uterus. - * thresholdPerc: Threshold in percentage of total mutations in a sample, only exposures larger than THRPERC are considered. If - * not specified 5. - * thresholdPval: P-value to determine the empirical probability that the exposure is lower than the threshold. If not specified - * then 0.05. - * maxRareSigs: Maximum number of rare signatures that are allowed to be present in each sample. If not specified 1. - * signaturesFile: The file name containing mutational signatures. Each signature is in a column, with signature names as column - * hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to - * provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures. - * rareSignaturesFile: The file name containing mutational signatures. Each signature is in a column, with signature names as - * column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use only to - * provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures. + * msId: Signature ID. + * msDescription: Signature description. * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ @@ -592,7 +591,8 @@ public RestResponse queryMutationalSignature(ObjectMap params) throws /** * Run mutational signature analysis for a given sample. - * @param data Mutational signature analysis params. + * @param data Mutational signature analysis parameters to index the genome context for that sample, and to compute both catalogue + * counts and signature fitting. In order to skip one of them, , use the following keywords: , catalogue, fitting. * @param params Map containing any of the following optional parameters. * study: Study [[user@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. @@ -865,7 +865,9 @@ public RestResponse runSampleEligibility(SampleEligibilityAnalysisParams da /** * Run quality control (QC) for a given sample. It includes variant stats, and if the sample is somatic, mutational signature and * genome plot are calculated. - * @param data Sample QC analysis params. Mutational signature and genome plot are calculated for somatic samples only. + * @param data Sample QC analysis params. Mutational signature and genome plot are calculated for somatic samples only. In order to + * skip some metrics, use the following keywords (separated by commas): variant-stats, signature, signature-catalogue, + * signature-fitting, genome-plot. * @param params Map containing any of the following optional parameters. * study: Study [[user@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java index a43e8b17097..0357d6e3b1f 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2022 OpenCB +* Copyright 2015-2023 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,7 +50,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2022-12-13 18:29:33 +* Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -59,7 +59,7 @@ /** * This class contains methods for the VariantOperation webservices. - * Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + * Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] * PATH: operation */ public class VariantOperationClient extends AbstractParentClient { diff --git a/opencga-client/src/main/javascript/Admin.js b/opencga-client/src/main/javascript/Admin.js index c035f28cf38..aaf4eb67fee 100644 --- a/opencga-client/src/main/javascript/Admin.js +++ b/opencga-client/src/main/javascript/Admin.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Alignment.js b/opencga-client/src/main/javascript/Alignment.js index b5666abb104..bd34f2fd0d0 100644 --- a/opencga-client/src/main/javascript/Alignment.js +++ b/opencga-client/src/main/javascript/Alignment.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -188,10 +188,9 @@ export default class Alignment extends OpenCGAParentClass { return this._post("analysis", null, "alignment/picard", null, "run", data, params); } - /** Compute quality control (QC) metrics for a given alignment file (including samtools stats, samtools flag stats, FastQC and HS metrics) - * @param {Object} data - Alignment quality control (QC) parameters. It computes: stats, flag stats, fastqc and hybrid-selection metrics. - * The BAM file is mandatory ever but the BED fileand the dictionary files are only mandatory for computing hybrid-selection (HS) - * metrics. In order to skip some metrics, use the following keywords (separated by commas): stats, flagstats, fastqc and hsmetrics. + /** Compute quality control (QC) metrics for a given alignment file: samtools stats, samtools flag stats and FastQC metrics. + * @param {Object} data - Alignment quality control (QC) parameters. It computes: stats, flag stats and fastqc metrics. The BAM file ID + * is mandatory and in order to skip some metrics, use the following keywords (separated by commas): stats, flagstats, fastqc. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - study. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not diff --git a/opencga-client/src/main/javascript/ClinicalAnalysis.js b/opencga-client/src/main/javascript/ClinicalAnalysis.js index 55410c7e4ca..de2bd3652ee 100644 --- a/opencga-client/src/main/javascript/ClinicalAnalysis.js +++ b/opencga-client/src/main/javascript/ClinicalAnalysis.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Cohort.js b/opencga-client/src/main/javascript/Cohort.js index 8c258e70f5f..02c5fd3faea 100644 --- a/opencga-client/src/main/javascript/Cohort.js +++ b/opencga-client/src/main/javascript/Cohort.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -104,8 +104,7 @@ export default class Cohort extends OpenCGAParentClass { * @param {String} [params.id] - Comma separated list of cohort IDs up to a maximum of 100. * @param {String} [params.name] - Comma separated list of cohort names up to a maximum of 100. * @param {String} [params.uuid] - Comma separated list of cohort IDs up to a maximum of 100. - * @param {"CASE_CONTROL CASE_SET CONTROL_SET PAIRED PAIRED_TUMOR AGGREGATE TIME_SERIES FAMILY TRIO COLLECTION"} [params.type] - Cohort - * type. + * @param {String} [params.type] - Cohort type. * @param {String} [params.creationDate] - creationDate. * @param {String} [params.modificationDate] - modificationDate. * @param {Boolean} [params.deleted] - deleted. @@ -165,8 +164,7 @@ export default class Cohort extends OpenCGAParentClass { * @param {String} [params.id] - Comma separated list of cohort IDs up to a maximum of 100. * @param {String} [params.name] - Comma separated list of cohort names up to a maximum of 100. * @param {String} [params.uuid] - Comma separated list of cohort IDs up to a maximum of 100. - * @param {"CASE_CONTROL CASE_SET CONTROL_SET PAIRED PAIRED_TUMOR AGGREGATE TIME_SERIES FAMILY TRIO COLLECTION"} [params.type] - Cohort - * type. + * @param {String} [params.type] - Cohort type. * @param {String} [params.creationDate] - creationDate. * @param {String} [params.modificationDate] - modificationDate. * @param {Boolean} [params.deleted] - deleted. diff --git a/opencga-client/src/main/javascript/DiseasePanel.js b/opencga-client/src/main/javascript/DiseasePanel.js index 32a8b7172d5..a4c5d719d32 100644 --- a/opencga-client/src/main/javascript/DiseasePanel.js +++ b/opencga-client/src/main/javascript/DiseasePanel.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Family.js b/opencga-client/src/main/javascript/Family.js index 111ff942c05..78666e9aa95 100644 --- a/opencga-client/src/main/javascript/Family.js +++ b/opencga-client/src/main/javascript/Family.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/File.js b/opencga-client/src/main/javascript/File.js index e82b788350a..e51146db071 100644 --- a/opencga-client/src/main/javascript/File.js +++ b/opencga-client/src/main/javascript/File.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:33 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/GA4GH.js b/opencga-client/src/main/javascript/GA4GH.js index 3ea170ed9df..d0fb8aef94a 100644 --- a/opencga-client/src/main/javascript/GA4GH.js +++ b/opencga-client/src/main/javascript/GA4GH.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Individual.js b/opencga-client/src/main/javascript/Individual.js index 1ea2f95476a..a9295aa774a 100644 --- a/opencga-client/src/main/javascript/Individual.js +++ b/opencga-client/src/main/javascript/Individual.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index e0881525091..ec694d273b3 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:33 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Meta.js b/opencga-client/src/main/javascript/Meta.js index 33bf6302871..5f429cb236e 100644 --- a/opencga-client/src/main/javascript/Meta.js +++ b/opencga-client/src/main/javascript/Meta.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Project.js b/opencga-client/src/main/javascript/Project.js index db370d088f2..7ca7e3af39f 100644 --- a/opencga-client/src/main/javascript/Project.js +++ b/opencga-client/src/main/javascript/Project.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:33 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Sample.js b/opencga-client/src/main/javascript/Sample.js index f75f6a31f02..232a12f3cb2 100644 --- a/opencga-client/src/main/javascript/Sample.js +++ b/opencga-client/src/main/javascript/Sample.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index 8073882c593..bbf659571ac 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:33 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/User.js b/opencga-client/src/main/javascript/User.js index 0471894b4dd..83cd0b83523 100644 --- a/opencga-client/src/main/javascript/User.js +++ b/opencga-client/src/main/javascript/User.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:33 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Variant.js b/opencga-client/src/main/javascript/Variant.js index f8350a862b9..05f934983fd 100644 --- a/opencga-client/src/main/javascript/Variant.js +++ b/opencga-client/src/main/javascript/Variant.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -292,6 +292,21 @@ export default class Variant extends OpenCGAParentClass { return this._post("analysis", null, "variant/gwas", null, "run", data, params); } + /** Run HRDetect analysis for a given somatic sample. + * @param {Object} data - HRDetect analysis parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not + * provided. + * @param {String} [params.jobDescription] - Job description. + * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. + * @param {String} [params.jobTags] - Job tags. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + runHrDetect(data, params) { + return this._post("analysis", null, "variant/hrDetect", null, "run", data, params); + } + /** [DEPRECATED] Use operation/variant/index * @param {Object} data - Variant index params. * @param {Object} [params] - The Object containing the following optional parameters: @@ -425,6 +440,7 @@ export default class Variant extends OpenCGAParentClass { * @param {String} [params.study] - Filter variants from the given studies, these can be either the numeric ID or the alias with the * format user@project:study. * @param {String} [params.sample] - Sample name. + * @param {String} [params.type] - Variant type. Valid values: SNV, SV. * @param {String} [params.ct] - List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases * 'loss_of_function' and 'protein_altering'. * @param {String} [params.biotype] - List of biotypes, e.g. protein_coding. @@ -451,31 +467,8 @@ export default class Variant extends OpenCGAParentClass { * both, oncogene, tumorSuppressorGene, fusion ]. * @param {Boolean} [params.panelIntersection] - Intersect panel genes and regions with given genes and regions from que input query. * This will prevent returning variants from regions out of the panel. - * @param {String} [params.catalogues] - File name containing mutational catalogues. Each sample catalogue is in a column, with sample - * names as column headers and channel. - * @param {String} [params.cataloguesContent] - Mutational catalogues. Each sample catalogue is in a column, with sample names as column - * headers and channel. - * @param {String} [params.fitMethod = "FitMS"] - Either Fit or FitMS. If not specified then FitMS. The default value is FitMS. - * @param {Number} [params.nBoot] - Number of bootstrap to be used. - * @param {String} [params.sigVersion = "RefSigv2"] - Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not specified RefSigv2. The - * default value is RefSigv2. - * @param {String} [params.organ] - When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific signatures will be used. If SIGVERSION - * is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be used. Available organs depend on the - * selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, - * Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, - * Stomach, Uterus. - * @param {Number} [params.thresholdPerc = "5f"] - Threshold in percentage of total mutations in a sample, only exposures larger than - * THRPERC are considered. If not specified 5. The default value is 5f. - * @param {Number} [params.thresholdPval = "0.05f"] - P-value to determine the empirical probability that the exposure is lower than the - * threshold. If not specified then 0.05. The default value is 0.05f. - * @param {Number} [params.maxRareSigs = "1"] - Maximum number of rare signatures that are allowed to be present in each sample. If not - * specified 1. The default value is 1. - * @param {String} [params.signaturesFile] - The file name containing mutational signatures. Each signature is in a column, with - * signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use - * only to provide your own signatures. When fitmethod=FitMS, these signatures are considered common signatures. - * @param {String} [params.rareSignaturesFile] - The file name containing mutational signatures. Each signature is in a column, with - * signature names as column hearders and channel names as row names in the first column with no header. Each column must sum to 1. Use - * only to provide your own signatures. When fitmethod=FitMS, these signatures are considered rare signatures. + * @param {String} [params.msId] - Signature ID. + * @param {String} [params.msDescription] - Signature description. * @returns {Promise} Promise object in the form of RestResponse instance. */ queryMutationalSignature(params) { @@ -483,7 +476,8 @@ export default class Variant extends OpenCGAParentClass { } /** Run mutational signature analysis for a given sample. - * @param {Object} data - Mutational signature analysis params. + * @param {Object} data - Mutational signature analysis parameters to index the genome context for that sample, and to compute both + * catalogue counts and signature fitting. In order to skip one of them, , use the following keywords: , catalogue, fitting. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not @@ -747,7 +741,9 @@ export default class Variant extends OpenCGAParentClass { /** Run quality control (QC) for a given sample. It includes variant stats, and if the sample is somatic, mutational signature and genome * plot are calculated. - * @param {Object} data - Sample QC analysis params. Mutational signature and genome plot are calculated for somatic samples only. + * @param {Object} data - Sample QC analysis params. Mutational signature and genome plot are calculated for somatic samples only. In + * order to skip some metrics, use the following keywords (separated by commas): variant-stats, signature, signature-catalogue, + * signature-fitting, genome-plot. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - Study [[user@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not diff --git a/opencga-client/src/main/javascript/VariantOperation.js b/opencga-client/src/main/javascript/VariantOperation.js index e18c289602e..5526811859b 100644 --- a/opencga-client/src/main/javascript/VariantOperation.js +++ b/opencga-client/src/main/javascript/VariantOperation.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2022-12-13 18:29:34 + * Autogenerated on: 2023-01-10 10:57:29 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py index f8becd9e23d..cea1a548ae1 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Admin(_ParentRestClient): """ This class contains methods for the 'Admin' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/admin """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py index 604dfe76a9a..aacfb1d3093 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Alignment(_ParentRestClient): """ This class contains methods for the 'Analysis - Alignment' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/analysis/alignment """ @@ -227,17 +227,14 @@ def run_picard(self, data=None, **options): def run_qc(self, data=None, **options): """ - Compute quality control (QC) metrics for a given alignment file - (including samtools stats, samtools flag stats, FastQC and HS - metrics). + Compute quality control (QC) metrics for a given alignment file: + samtools stats, samtools flag stats and FastQC metrics. PATH: /{apiVersion}/analysis/alignment/qc/run :param dict data: Alignment quality control (QC) parameters. It - computes: stats, flag stats, fastqc and hybrid-selection metrics. - The BAM file is mandatory ever but the BED fileand the dictionary - files are only mandatory for computing hybrid-selection (HS) - metrics. In order to skip some metrics, use the following keywords - (separated by commas): stats, flagstats, fastqc and hsmetrics. + computes: stats, flag stats and fastqc metrics. The BAM file ID is + mandatory and in order to skip some metrics, use the following + keywords (separated by commas): stats, flagstats, fastqc. (REQUIRED) :param str study: study. :param str job_id: Job ID. It must be a unique string within the diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py index 14851de1308..a16bffea483 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class ClinicalAnalysis(_ParentRestClient): """ This class contains methods for the 'Analysis - Clinical' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/analysis/clinical """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py index 10b74ae57fb..fb5ab050ffc 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Cohort(_ParentRestClient): """ This class contains methods for the 'Cohorts' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/cohorts """ @@ -126,9 +126,7 @@ def distinct(self, field, **options): of 100. :param str uuid: Comma separated list of cohort IDs up to a maximum of 100. - :param str type: Cohort type. Allowed values: ['CASE_CONTROL CASE_SET - CONTROL_SET PAIRED PAIRED_TUMOR AGGREGATE TIME_SERIES FAMILY TRIO - COLLECTION'] + :param str type: Cohort type. :param str creation_date: creationDate. :param str modification_date: modificationDate. :param bool deleted: deleted. @@ -207,9 +205,7 @@ def search(self, **options): of 100. :param str uuid: Comma separated list of cohort IDs up to a maximum of 100. - :param str type: Cohort type. Allowed values: ['CASE_CONTROL CASE_SET - CONTROL_SET PAIRED PAIRED_TUMOR AGGREGATE TIME_SERIES FAMILY TRIO - COLLECTION'] + :param str type: Cohort type. :param str creation_date: creationDate. :param str modification_date: modificationDate. :param bool deleted: deleted. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py index 285f6a89b36..50ab7cf987f 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class DiseasePanel(_ParentRestClient): """ This class contains methods for the 'Disease Panels' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/panels """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py index f7226d9d148..9e11abb3e8d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Family(_ParentRestClient): """ This class contains methods for the 'Families' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/families """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py index 15cfc074327..a37621a95fb 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class File(_ParentRestClient): """ This class contains methods for the 'Files' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/files """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py index 483b1b9efb2..2f834140099 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class GA4GH(_ParentRestClient): """ This class contains methods for the 'GA4GH' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/ga4gh """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py index 261e79076bb..55faf7364d3 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Individual(_ParentRestClient): """ This class contains methods for the 'Individuals' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/individuals """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py index 078cbc85fbb..f14d49ae72e 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Job(_ParentRestClient): """ This class contains methods for the 'Jobs' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/jobs """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py index fafb211a09b..752d31671e8 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Meta(_ParentRestClient): """ This class contains methods for the 'Meta' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/meta """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py index ccaa5614a5a..d64f68af645 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Project(_ParentRestClient): """ This class contains methods for the 'Projects' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/projects """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py index 2cf07c5dfaf..dd1725d2f2c 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Sample(_ParentRestClient): """ This class contains methods for the 'Samples' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/samples """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 114a017e211..b400e064da5 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Study(_ParentRestClient): """ This class contains methods for the 'Studies' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/studies """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py index 91ecf00e4fb..d52ca1b64fa 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class User(_ParentRestClient): """ This class contains methods for the 'Users' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/users """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py index cd220ab8a2b..619fda87a3d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Variant(_ParentRestClient): """ This class contains methods for the 'Analysis - Variant' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/analysis/variant """ @@ -365,6 +365,24 @@ def run_gwas(self, data=None, **options): return self._post(category='analysis', resource='run', subcategory='variant/gwas', data=data, **options) + def run_hr_detect(self, data=None, **options): + """ + Run HRDetect analysis for a given somatic sample. + PATH: /{apiVersion}/analysis/variant/hrDetect/run + + :param dict data: HRDetect analysis parameters. (REQUIRED) + :param str study: Study [[user@]project:]study where study and project + can be either the ID or UUID. + :param str job_id: Job ID. It must be a unique string within the + study. An ID will be autogenerated automatically if not provided. + :param str job_description: Job description. + :param str job_depends_on: Comma separated list of existing job IDs + the job will depend on. + :param str job_tags: Job tags. + """ + + return self._post(category='analysis', resource='run', subcategory='variant/hrDetect', data=data, **options) + def run_index(self, data=None, **options): """ [DEPRECATED] Use operation/variant/index. @@ -539,6 +557,7 @@ def query_mutational_signature(self, **options): either the numeric ID or the alias with the format user@project:study. :param str sample: Sample name. + :param str type: Variant type. Valid values: SNV, SV. :param str ct: List of SO consequence types, e.g. missense_variant,stop_lost or SO:0001583,SO:0001578. Accepts aliases 'loss_of_function' and 'protein_altering'. @@ -575,46 +594,8 @@ def query_mutational_signature(self, **options): :param bool panel_intersection: Intersect panel genes and regions with given genes and regions from que input query. This will prevent returning variants from regions out of the panel. - :param str catalogues: File name containing mutational catalogues. - Each sample catalogue is in a column, with sample names as column - headers and channel. - :param str catalogues_content: Mutational catalogues. Each sample - catalogue is in a column, with sample names as column headers and - channel. - :param str fit_method: Either Fit or FitMS. If not specified then - FitMS. - :param int n_boot: Number of bootstrap to be used. - :param str sig_version: Either COSMICv2, COSMICv3.2, RefSigv1 or - RefSigv2. If not specified RefSigv2. - :param str organ: When using RefSigv1 or RefSigv2 as SIGVERSION, - organ-specific signatures will be used. If SIGVERSION is COSMICv2 - or COSMICv3.2, then a selection of signatures found in the given - organ will be used. Available organs depend on the selected - SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, - Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, - Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), - Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, - Stomach, Uterus. - :param float threshold_perc: Threshold in percentage of total - mutations in a sample, only exposures larger than THRPERC are - considered. If not specified 5. - :param float threshold_pval: P-value to determine the empirical - probability that the exposure is lower than the threshold. If not - specified then 0.05. - :param int max_rare_sigs: Maximum number of rare signatures that are - allowed to be present in each sample. If not specified 1. - :param str signatures_file: The file name containing mutational - signatures. Each signature is in a column, with signature names as - column hearders and channel names as row names in the first column - with no header. Each column must sum to 1. Use only to provide your - own signatures. When fitmethod=FitMS, these signatures are - considered common signatures. - :param str rare_signatures_file: The file name containing mutational - signatures. Each signature is in a column, with signature names as - column hearders and channel names as row names in the first column - with no header. Each column must sum to 1. Use only to provide your - own signatures. When fitmethod=FitMS, these signatures are - considered rare signatures. + :param str ms_id: Signature ID. + :param str ms_description: Signature description. """ return self._get(category='analysis', resource='query', subcategory='variant/mutationalSignature', **options) @@ -624,7 +605,10 @@ def run_mutational_signature(self, data=None, **options): Run mutational signature analysis for a given sample. PATH: /{apiVersion}/analysis/variant/mutationalSignature/run - :param dict data: Mutational signature analysis params. (REQUIRED) + :param dict data: Mutational signature analysis parameters to index + the genome context for that sample, and to compute both catalogue + counts and signature fitting. In order to skip one of them, , use + the following keywords: , catalogue, fitting. (REQUIRED) :param str study: Study [[user@]project:]study where study and project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the @@ -1009,7 +993,10 @@ def run_sample_qc(self, data=None, **options): PATH: /{apiVersion}/analysis/variant/sample/qc/run :param dict data: Sample QC analysis params. Mutational signature and - genome plot are calculated for somatic samples only. (REQUIRED) + genome plot are calculated for somatic samples only. In order to + skip some metrics, use the following keywords (separated by + commas): variant-stats, signature, signature-catalogue, + signature-fitting, genome-plot. (REQUIRED) :param str study: Study [[user@]project:]study where study and project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py index e4e768edcf7..a13b399fa3b 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2022-12-13 18:29:33 + Autogenerated on: 2023-01-10 10:57:29 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class VariantOperation(_ParentRestClient): """ This class contains methods for the 'Operations - Variant Storage' webservices - Client version: 2.4.12-SNAPSHOT [8eb55dfca0127d68feb6dee977bc257c1f82354d] + Client version: 2.6.0-SNAPSHOT [cd4c96caf1cb7afcdec5f54a7de72bde4f63820e] PATH: /{apiVersion}/operation """ diff --git a/opencga-clinical/pom.xml b/opencga-clinical/pom.xml index 618281c5d5c..ef0faea26dc 100644 --- a/opencga-clinical/pom.xml +++ b/opencga-clinical/pom.xml @@ -5,7 +5,7 @@ org.opencb.opencga opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/opencga-core/pom.xml b/opencga-core/pom.xml index 46f40c5a234..dc09ecdc865 100644 --- a/opencga-core/pom.xml +++ b/opencga-core/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index 0962295831d..f768d2ea006 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -1,10 +1,8 @@ package org.opencb.opencga.core.api; -import com.beust.jcommander.DynamicParameter; -import com.beust.jcommander.Parameter; - -import java.util.HashMap; -import java.util.Map; +import org.opencb.opencga.core.models.alignment.AlignmentQcParams; +import org.opencb.opencga.core.models.variant.MutationalSignatureAnalysisParams; +import org.opencb.opencga.core.models.variant.SampleQcAnalysisParams; public class FieldConstants { @@ -30,7 +28,8 @@ public class FieldConstants { public static final String GENERIC_CUSTOM_STATUS = "Object to set a custom status."; public static final String GENERIC_INTERNAL = "Internal field to manage the object."; public static final String GENERIC_NAME = "Name of the ."; - public static final String GENERIC_ID_DESCRIPTION = "Object ID is a mandatory parameter when creating a new one, this ID cannot be changed at the moment."; + public static final String GENERIC_ID_DESCRIPTION = "Object ID is a mandatory parameter when creating a new one, this ID cannot be" + + " changed at the moment."; public static final String GENERIC_STATUS_DESCRIPTION = "Object status."; public static final String GENERIC_STATS = "Stats of the object."; //QualityControl @@ -74,12 +73,20 @@ public class FieldConstants { public static final String SAMPLE_QUALITY_CONTROL_FILES_DESCRIPTION = "Files used for the quality control of the sample."; public static final String SAMPLE_QUALITY_CONTROL_COMMENTS_DESCRIPTION = "Comments for the quality control of the sample."; public static final String SAMPLE_QUALITY_CONTROL_VARIANT_DESCRIPTION = "Describes variant quality control."; + public static final String SAMPLE_QUALITY_CONTROL_SKIP_DESCRIPTION = "Quality control metrics to skip. Valid values are: " + + SampleQcAnalysisParams.VARIANT_STATS_SKIP_VALUE + ", " + SampleQcAnalysisParams.SIGNATURE_SKIP_VALUE + ", " + + SampleQcAnalysisParams.SIGNATURE_CATALOGUE_SKIP_VALUE + ", " + SampleQcAnalysisParams.SIGNATURE_FITTING_SKIP_VALUE + ", " + + SampleQcAnalysisParams.GENOME_PLOT_SKIP_VALUE; + public static final String SAMPLE_QUALITY_CONTROL_OVERWRITE_DESCRIPTION = "Overwrite sample quality control in OpenCGA catalog."; //SampleVariantQualityControlMetrics - public static final String SAMPLE_QUALITY_CONTROL_METRICS_VARIANT_STATS_DESCRIPTION = "Variant stats for the quality control of the " - + "sample."; - public static final String SAMPLE_QUALITY_CONTROL_METRICS_SIGNATURES_DESCRIPTION = "Signature for the quality control of the sample."; - public static final String SAMPLE_QUALITY_CONTROL_METRICS_GENOME_PLOT_DESCRIPTION = "Genome plot for the quality control of the sample."; + public static final String SAMPLE_QUALITY_CONTROL_METRICS_VARIANT_STATS_DESCRIPTION = "Variant stats for the quality control of the" + + " sample."; + public static final String SAMPLE_QUALITY_CONTROL_METRICS_SIGNATURES_DESCRIPTION = "List of signature for the quality control of the" + + " sample."; + public static final String SAMPLE_QUALITY_CONTROL_METRICS_GENOME_PLOT_DESCRIPTION = "Genome plot for the quality control of the" + + " sample."; + public static final String SAMPLE_QUALITY_CONTROL_METRICS_HRDETEC_DESCRIPTION = "List of HRDetect results for a given sample"; public static final String SAMPLE_QUALITY_CONTROL_METRICS_FILES_DESCRIPTION = "File for the quality control metrics of the " + "sample."; @@ -131,16 +138,19 @@ public class FieldConstants { + "true or false."; //Family - public static final String FAMILY_ID_DESCRIPTION = "Family is a mandatory parameter when creating a new sample, this ID cannot be changed at the moment."; + public static final String FAMILY_ID_DESCRIPTION = "Family is a mandatory parameter when creating a new sample, this ID cannot be" + + " changed at the moment."; public static final String FAMILY_NAME = "Family name."; public static final String FAMILY_MEMBERS = "List of individuals who are family members."; public static final String FAMILY_DISORDERS = "Family disorders."; public static final String FAMILY_EXPECTED_SIZE = "Family expected size."; - public static final String FAMILY_ROLES = "Map of members ids and enum of roles (FATHER, MOTHER, IDENTICAL_TWIN, SON, UNCLE, PATERNAL_GRANDFATHER.) ."; + public static final String FAMILY_ROLES = "Map of members ids and enum of roles (FATHER, MOTHER, IDENTICAL_TWIN, SON, UNCLE," + + " PATERNAL_GRANDFATHER.) ."; //FamilyQualityControl public static final String FAMILY_QUALITY_CONTROL_RELATEDNESS_DESCRIPTION = "Reports of family relationship."; - public static final String INDIVIDUAL_QUALITY_CONTROL_INFERRED_SEX_REPORT_DESCRIPTION = "List of inferred sex reports, it depends on the method (currently by coverage ratio)."; + public static final String INDIVIDUAL_QUALITY_CONTROL_INFERRED_SEX_REPORT_DESCRIPTION = "List of inferred sex reports, it depends on" + + " the method (currently by coverage ratio)."; public static final String INDIVIDUAL_QUALITY_CONTROL_SAMPLE_RELATEDNESS_REPORT_DESCRIPTION = "Reports of samples relatedness."; public static final String INDIVIDUAL_QUALITY_CONTROL_MENDELIAN_ERRORS_DESCRIPTION = "Mendelian errors."; @@ -395,43 +405,73 @@ public class FieldConstants { // Mutational signature (sample-qc-run, mutationsl-signature-run, mutational-signature-query) public static final String MUTATIONAL_SIGNATURE_ID_DESCRIPTION = "Signature ID."; public static final String MUTATIONAL_SIGNATURE_DESCRIPTION_DESCRIPTION = "Signature description."; + public static final String MUTATIONAL_SIGNATURE_SAMPLE_DESCRIPTION = "Sample."; public static final String MUTATIONAL_SIGNATURE_QUERY_DESCRIPTION = "Signature query in JSON format, e.g: \"\"{\\\"sample\\\":" + "\\\"NR123456_T\\\", \\\"fileData\\\": \\\"NR.123456_T_vs_NR.1234567_G.annot.vcf.gz:FILTER=PASS;CLPM<=0;ASMD>=140\\\"}\"."; public static final String MUTATIONAL_SIGNATURE_CATALOGUES_DESCRIPTION = "File name containing mutational catalogues. Each sample" + " catalogue is in a column, with sample names as column headers and channel."; public static final String MUTATIONAL_SIGNATURE_CATALOGUES_CONTENT_DESCRIPTION = "Mutational catalogues. Each sample catalogue is in a" + " column, with sample names as column headers and channel."; + public static final String MUTATIONAL_SIGNATURE_FIT_ID_DESCRIPTION = "Fiiting signature ID"; public static final String MUTATIONAL_SIGNATURE_FIT_METHOD_DESCRIPTION = "Either Fit or FitMS. If not specified then FitMS"; - public static final String MUTATIONAL_SIGNATURE_N_BOOT_DESCRIPTION = "Number of bootstrap to be used."; - public static final String MUTATIONAL_SIGNATURE_SIG_VERSION_DESCRIPTION = "Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2. If not" - + " specified RefSigv2."; - public static final String MUTATIONAL_SIGNATURE_ORGAN_DESCRIPTION = "When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific" - + " signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ will be" - + " used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder, Bone_SoftTissue, Breast," - + " Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET (v2 only), Oral_Oropharyngeal" - + " (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus."; - public static final String MUTATIONAL_SIGNATURE_THRESHOLD_PERC_DESCRIPTION = "Threshold in percentage of total mutations in a sample," - + " only exposures larger than THRPERC are considered. If not specified 5."; - public static final String MUTATIONAL_SIGNATURE_THRESHOLD_PVAL_DESCRIPTION = "P-value to determine the empirical probability that the" - + " exposure is lower than the threshold. If not specified then 0.05."; - public static final String MUTATIONAL_SIGNATURE_MAX_RARE_SIGS_DESCRIPTION = "Maximum number of rare signatures that are allowed to be" - + " present in each sample. If not specified 1."; - public static final String MUTATIONAL_SIGNATURE_SIGNATURES_FILE_DESCRIPTION = "The file name containing mutational signatures. Each" + public static final String MUTATIONAL_SIGNATURE_FIT_N_BOOT_DESCRIPTION = "Number of bootstrap to be used."; + public static final String MUTATIONAL_SIGNATURE_FIT_SIG_VERSION_DESCRIPTION = "Either COSMICv2, COSMICv3.2, RefSigv1 or RefSigv2." + + " If not specified RefSigv2."; + public static final String MUTATIONAL_SIGNATURE_FIT_ORGAN_DESCRIPTION = "When using RefSigv1 or RefSigv2 as SIGVERSION, organ-specific" + + " signatures will be used. If SIGVERSION is COSMICv2 or COSMICv3.2, then a selection of signatures found in the given organ" + + " will be used. Available organs depend on the selected SIGVERSION. For RefSigv1 or RefSigv2: Biliary, Bladder," + + " Bone_SoftTissue, Breast, Cervix (v1 only), CNS, Colorectal, Esophagus, Head_neck, Kidney, Liver, Lung, Lymphoid, NET" + + " (v2 only), Oral_Oropharyngeal (v2 only), Ovary, Pancreas, Prostate, Skin, Stomach, Uterus."; + public static final String MUTATIONAL_SIGNATURE_FIT_THRESHOLD_PERC_DESCRIPTION = "Threshold in percentage of total mutations in a" + + " sample, only exposures larger than THRPERC are considered. If not specified 5."; + public static final String MUTATIONAL_SIGNATURE_FIT_THRESHOLD_PVAL_DESCRIPTION = "P-value to determine the empirical probability that" + + " the exposure is lower than the threshold. If not specified then 0.05."; + public static final String MUTATIONAL_SIGNATURE_FIT_MAX_RARE_SIGS_DESCRIPTION = "Maximum number of rare signatures that are allowed to" + + " be present in each sample. If not specified 1."; + public static final String MUTATIONAL_SIGNATURE_FIT_SIGNATURES_FILE_DESCRIPTION = "The file name containing mutational signatures. Each" + " signature is in a column, with signature names as column hearders and channel names as row names in the first column with" + " no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures are" + " considered common signatures."; - public static final String MUTATIONAL_SIGNATURE_RARE_SIGNATURES_FILE_DESCRIPTION = "The file name containing mutational signatures." + public static final String MUTATIONAL_SIGNATURE_FIT_RARE_SIGNATURES_FILE_DESCRIPTION = "The file name containing mutational signatures." + " Each signature is in a column, with signature names as column hearders and channel names as row names in the first column" + " with no header. Each column must sum to 1. Use only to provide your own signatures. When fitmethod=FitMS, these signatures" + " are considered rare signatures."; + public static final String MUTATIONAL_SIGNATURE_SKIP_DESCRIPTION = "To skip to compute catalogue counts or the signature fitting. Use " + + " the following keywords: " + MutationalSignatureAnalysisParams.SIGNATURE_CATALOGUE_SKIP_VALUE + ", " + + MutationalSignatureAnalysisParams.SIGNATURE_FITTING_SKIP_VALUE + "."; + + // HRDetect analysis + public static final String HRDETECT_ID_DESCRIPTION = "ID to identify the HRDetect results."; + public static final String HRDETECT_DESCRIPTION_DESCRIPTION = "Decription for these particular HRDetect results."; + public static final String HRDETECT_SNV_FITTING_ID_DESCRIPTION = "Mutational signature fitting ID for SNV."; + public static final String HRDETECT_SV_FITTING_ID_DESCRIPTION = "Mutational signature fitting ID for SV."; + public static final String HRDETECT_CNV_QUERY_DESCRIPTION = "CNV query"; + public static final String HRDETECT_INDEL_QUERY_DESCRIPTION = "INDEL query"; + public static final String HRDETECT_SNV3_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SNV3 input for" + + " HRDetect."; + public static final String HRDETECT_SNV8_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SNV8 input for" + + " HRDetect."; + public static final String HRDETECT_SV3_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SV3 input for" + + " HRDetect."; + public static final String HRDETECT_SV8_CUSTOM_NAME_DESCRIPTION = "Custom signature name that will be considered as SV8 input for" + + " HRDetect."; + public static final String HRDETECT_BOOTSTRAP_DESCRIPTION = "Request HRDetect with bootstrap."; // Genome plot (sample-qc-run) public static final String GENOME_PLOT_ID_DESCRIPTION = "Genome plot ID."; public static final String GENOME_PLOT_DESCRIPTION_DESCRIPTION = "Genome plot description."; public static final String GENOME_PLOT_CONFIGURATION_FILE_DESCRIPTION = "Genome plot configuration file."; - // Varaint stats (sample-qc-run) + // Variant stats (sample-qc-run) public static final String VARIANT_STATS_ID_DESCRIPTION = "Variant stats ID."; public static final String VARIANT_STATS_DESCRIPTION_DESCRIPTION = "Variant stats description."; public static final String VARIANT_STATS_QUERY_DESCRIPTION = "Variant stats query in JSON format."; + + // Alignment QC analysis (asample-qc-run) + public static final String ALIGNMENT_QC_BAM_FILE_DESCRIPTION = "ID for the BAM file to process."; + public static final String ALIGNMENT_QC_SKIP_DESCRIPTION = "To skip any alignment QC metrics use the following keywords (separated by" + + " commas): " + AlignmentQcParams.STATS_SKIP_VALUE + ", " + AlignmentQcParams.FLAGSTATS_SKIP_VALUE + ", " + + AlignmentQcParams.FASTQC_METRICS_SKIP_VALUE; + public static final String ALIGNMENT_QC_OVERWRITE_DESCRIPTION = "To overwrite the QC metrics already computed."; + } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index 8004d2a504e..a6fa1a2af32 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -74,7 +74,8 @@ public class ParamConstants { public static final String CELLBASE_URL = "https://ws.zettagenomics.com/cellbase"; - public static final String CELLBASE_VERSION = "v5"; + public static final String CELLBASE_VERSION = "v5.1"; + public static final String CELLBASE_DATA_RELEASE = "2"; public static final String POP_FREQ_1000G_CB_V4 = "1kG_phase3"; public static final String POP_FREQ_1000G_CB_V5 = "1000G"; @@ -579,8 +580,8 @@ public class ParamConstants { public static final String ALIGNMENT_COVERAGE_RATIO_DESCRIPTION = "Compute coverage ratio from file #1 vs file #2, (e.g. somatic vs " + "germline)"; // --------------------------------------------- - public static final String ALIGNMENT_QC_DESCRIPTION = "Compute quality control (QC) metrics for a given alignment file (including " + - "samtools stats, samtools flag stats, FastQC and HS metrics)"; + public static final String ALIGNMENT_QC_DESCRIPTION = "Compute quality control (QC) metrics for a given alignment file:" + + " samtools stats, samtools flag stats and FastQC metrics."; public static final String ALIGNMENT_STATS_DESCRIPTION = "Compute stats (based on samtools/stats command) for a given alignment file"; public static final String ALIGNMENT_FLAG_STATS_DESCRIPTION = "Compute flag stats (based on samtools/flagstat command) for a given " + "alignment file"; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java index 96544c6ad81..a8000e50e7f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java @@ -16,35 +16,34 @@ package org.opencb.opencga.core.config.storage; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.commons.lang3.StringUtils; import org.opencb.cellbase.client.config.ClientConfiguration; import org.opencb.cellbase.client.config.RestConfig; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.ParamConstants; import org.slf4j.LoggerFactory; import java.util.Collections; -import java.util.List; /** * Created by imedina on 04/05/15. */ @JsonIgnoreProperties(allowSetters = true, value = {"host", "preferred", "hosts", "database"}) public class CellBaseConfiguration { - /* - * URL to CellBase REST web services, by default official UCam installation is used - */ + + @DataField(id = "url", description = "URL to CellBase REST web services, by default official ZettaGenomics installation is used") private String url; - /* - * CellBase version to be used, by default the 'v4' stable - */ + @DataField(id = "version", description = "URL to CellBase REST web services, by default official ZettaGenomics installation is used") private String version; - private static final String CELLBASE_HOST = "http://ws.opencb.org/cellbase/"; - private static final String CELLBASE_VERSION = "v4"; + @DataField(id = "dataRelease", description = "CellBase data release version to be used. If empty, will use the active one") + private String dataRelease; public CellBaseConfiguration() { - this(CELLBASE_HOST, CELLBASE_VERSION); + this(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION); } public CellBaseConfiguration(String url, String version) { @@ -52,11 +51,18 @@ public CellBaseConfiguration(String url, String version) { this.version = version; } + public CellBaseConfiguration(String url, String version, String dataRelease) { + this.url = url; + this.version = version; + this.dataRelease = dataRelease; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("CellBaseConfiguration{"); - sb.append("url=").append(url); + sb.append("url='").append(url).append('\''); sb.append(", version='").append(version).append('\''); + sb.append(", dataRelease='").append(dataRelease).append('\''); sb.append('}'); return sb.toString(); } @@ -70,41 +76,6 @@ public CellBaseConfiguration setUrl(String url) { return this; } -// @Deprecated -// public String getHost() { -// return url; -// } -// -// @Deprecated -// public CellBaseConfiguration setHost(String host) { -// if (host != null) { -// LoggerFactory.getLogger(CellBaseConfiguration.class).warn("Deprecated option 'cellbase.host'. Use 'cellbase.url'"); -// } -// url = host; -// return this; -// } - -// @Deprecated -// public List getHosts() { -// return Collections.singletonList(url); -// } -// -// @Deprecated -// public CellBaseConfiguration setHosts(List hosts) { -// if (hosts != null) { -// LoggerFactory.getLogger(CellBaseConfiguration.class).warn("Deprecated option 'cellbase.hosts'. Use 'cellbase.url'"); -// } -// if (hosts == null || hosts.isEmpty()) { -// url = null; -// } else { -// if (hosts.size() != 1) { -// throw new IllegalArgumentException("Unsupported multiple cellbase hosts"); -// } -// url = hosts.get(0); -// } -// return this; -// } - public String getVersion() { return version; } @@ -114,12 +85,23 @@ public CellBaseConfiguration setVersion(String version) { return this; } + public String getDataRelease() { + return dataRelease; + } + + public CellBaseConfiguration setDataRelease(String dataRelease) { + this.dataRelease = dataRelease; + return this; + } + @Deprecated + @JsonIgnore public Object getDatabase() { return null; } @Deprecated + @JsonIgnore public CellBaseConfiguration setDatabase(Object database) { if (database != null) { LoggerFactory.getLogger(CellBaseConfiguration.class).warn("Deprecated option 'storage-configuration.yml#cellbase.database'"); @@ -128,11 +110,13 @@ public CellBaseConfiguration setDatabase(Object database) { } @Deprecated + @JsonIgnore public String getPreferred() { return ""; } @Deprecated + @JsonIgnore public CellBaseConfiguration setPreferred(String preferred) { if (StringUtils.isNotEmpty(preferred)) { LoggerFactory.getLogger(CellBaseConfiguration.class).warn("Deprecated option 'storage-configuration.yml#cellbase.preferred'"); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentQcParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentQcParams.java index b4becf1deb0..16507adb547 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentQcParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/alignment/AlignmentQcParams.java @@ -1,32 +1,35 @@ package org.opencb.opencga.core.models.alignment; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.tools.ToolParams; public class AlignmentQcParams extends ToolParams { - public static String STATS_SKIP_VALUE = "stats"; - public static String FLAGSTATS_SKIP_VALUE = "flagstats"; - public static String FASTQC_METRICS_SKIP_VALUE = "fastqc"; - public static String HS_METRICS_SKIP_VALUE = "hsmetrics"; + public static final String STATS_SKIP_VALUE = "stats"; + public static final String FLAGSTATS_SKIP_VALUE = "flagstats"; + public static final String FASTQC_METRICS_SKIP_VALUE = "fastqc"; - public static final String DESCRIPTION = "Alignment quality control (QC) parameters. It computes: stats, flag stats, fastqc and" - + " hybrid-selection metrics. The BAM file is mandatory ever but the BED fileand the dictionary files are only mandatory for" - + " computing hybrid-selection (HS) metrics. In order to skip some metrics, use the following keywords (separated by commas): " - + "stats, flagstats, fastqc and hsmetrics"; + public static final String DESCRIPTION = "Alignment quality control (QC) parameters. It computes: stats, flag stats and fastqc metrics." + + " The BAM file ID is mandatory and in order to skip some metrics, use the following keywords (separated by commas): " + + STATS_SKIP_VALUE + ", " + FLAGSTATS_SKIP_VALUE + ", " + FASTQC_METRICS_SKIP_VALUE; + @DataField(id = "bamFile", description = FieldConstants.ALIGNMENT_QC_BAM_FILE_DESCRIPTION) private String bamFile; - private String bedFile; - private String dictFile; + + @DataField(id = "skip", description = FieldConstants.ALIGNMENT_QC_SKIP_DESCRIPTION) private String skip; + + @DataField(id = "overwrite", description = FieldConstants.ALIGNMENT_QC_OVERWRITE_DESCRIPTION) private boolean overwrite; + + @DataField(id = "outdir", description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) private String outdir; public AlignmentQcParams() { } - public AlignmentQcParams(String bamFile, String bedFile, String dictFile, String skip, boolean overwrite, String outdir) { + public AlignmentQcParams(String bamFile, String skip, boolean overwrite, String outdir) { this.bamFile = bamFile; - this.bedFile = bedFile; - this.dictFile = dictFile; this.skip = skip; this.overwrite = overwrite; this.outdir = outdir; @@ -36,8 +39,6 @@ public AlignmentQcParams(String bamFile, String bedFile, String dictFile, String public String toString() { final StringBuilder sb = new StringBuilder("AlignmentQcParams{"); sb.append("bamFile='").append(bamFile).append('\''); - sb.append(", bedFile='").append(bedFile).append('\''); - sb.append(", dictFile='").append(dictFile).append('\''); sb.append(", skip='").append(skip).append('\''); sb.append(", overwrite=").append(overwrite); sb.append(", outdir='").append(outdir).append('\''); @@ -54,24 +55,6 @@ public AlignmentQcParams setBamFile(String bamFile) { return this; } - public String getBedFile() { - return bedFile; - } - - public AlignmentQcParams setBedFile(String bedFile) { - this.bedFile = bedFile; - return this; - } - - public String getDictFile() { - return dictFile; - } - - public AlignmentQcParams setDictFile(String dictFile) { - this.dictFile = dictFile; - return this; - } - public String getSkip() { return skip; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleVariantQualityControlMetrics.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleVariantQualityControlMetrics.java index 189e46ec86f..66a3a31da05 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleVariantQualityControlMetrics.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleVariantQualityControlMetrics.java @@ -1,6 +1,7 @@ package org.opencb.opencga.core.models.sample; import org.opencb.biodata.models.clinical.qc.GenomePlot; +import org.opencb.biodata.models.clinical.qc.HRDetect; import org.opencb.biodata.models.clinical.qc.SampleQcVariantStats; import org.opencb.biodata.models.clinical.qc.Signature; import org.opencb.commons.annotations.DataField; @@ -23,19 +24,24 @@ public class SampleVariantQualityControlMetrics { description = FieldConstants.SAMPLE_QUALITY_CONTROL_METRICS_GENOME_PLOT_DESCRIPTION) private GenomePlot genomePlot; + @DataField(id = "hrDetects", + description = FieldConstants.SAMPLE_QUALITY_CONTROL_METRICS_HRDETEC_DESCRIPTION) + private List hrDetects;; + @DataField(id = "files", name = "files", description = FieldConstants.SAMPLE_QUALITY_CONTROL_METRICS_FILES_DESCRIPTION) private List files; public SampleVariantQualityControlMetrics() { - this(new ArrayList<>(), new ArrayList<>(), null, new ArrayList<>()); + this(new ArrayList<>(), new ArrayList<>(), null, new ArrayList<>(), new ArrayList<>()); } - public SampleVariantQualityControlMetrics(List variantStats, List signatures, - GenomePlot genomePlot, List files) { + public SampleVariantQualityControlMetrics(List variantStats, List signatures, GenomePlot genomePlot, + List hrDetects, List files) { this.variantStats = variantStats; this.signatures = signatures; this.genomePlot = genomePlot; + this.hrDetects = hrDetects; this.files = files; } @@ -45,6 +51,7 @@ public String toString() { sb.append("variantStats=").append(variantStats); sb.append(", signatures=").append(signatures); sb.append(", genomePlot=").append(genomePlot); + sb.append(", hrDetects=").append(hrDetects); sb.append(", files=").append(files); sb.append('}'); return sb.toString(); @@ -77,6 +84,15 @@ public SampleVariantQualityControlMetrics setGenomePlot(GenomePlot genomePlot) { return this; } + public List getHrDetects() { + return hrDetects; + } + + public SampleVariantQualityControlMetrics setHrDetects(List hrDetects) { + this.hrDetects = hrDetects; + return this; + } + public List getFiles() { return files; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/HRDetectAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/HRDetectAnalysisParams.java new file mode 100644 index 00000000000..370999129bf --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/HRDetectAnalysisParams.java @@ -0,0 +1,223 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.core.models.variant; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.tools.ToolParams; + +public class HRDetectAnalysisParams extends ToolParams { + public static final String DESCRIPTION = "HRDetect analysis parameters."; + + @DataField(id = "id", description = FieldConstants.HRDETECT_ID_DESCRIPTION) + private String id; + + @DataField(id = "description", description = FieldConstants.HRDETECT_DESCRIPTION_DESCRIPTION) + private String description; + + @DataField(id = "sampleId", description = FieldConstants.SAMPLE_ID_DESCRIPTION) + private String sampleId; + + @DataField(id = "svnFittingId", description = FieldConstants.HRDETECT_SNV_FITTING_ID_DESCRIPTION) + private String snvFittingId; + + @DataField(id = "svFittingId", description = FieldConstants.HRDETECT_SV_FITTING_ID_DESCRIPTION) + private String svFittingId; + + @DataField(id = "cnvQuery", description = FieldConstants.HRDETECT_CNV_QUERY_DESCRIPTION) + private String cnvQuery; + + @DataField(id = "indelQuery", description = FieldConstants.HRDETECT_INDEL_QUERY_DESCRIPTION) + private String indelQuery; + + @DataField(id = "snv3CustomName", description = FieldConstants.HRDETECT_SNV3_CUSTOM_NAME_DESCRIPTION) + private String snv3CustomName; + + @DataField(id = "snv8CustomName", description = FieldConstants.HRDETECT_SNV8_CUSTOM_NAME_DESCRIPTION) + private String snv8CustomName; + + @DataField(id = "sv3CustomName", description = FieldConstants.HRDETECT_SV3_CUSTOM_NAME_DESCRIPTION) + private String sv3CustomName; + + @DataField(id = "sv8CustomName", description = FieldConstants.HRDETECT_SV8_CUSTOM_NAME_DESCRIPTION) + private String sv8CustomName; + + @DataField(id = "bootstrap", description = FieldConstants.HRDETECT_BOOTSTRAP_DESCRIPTION) + private Boolean bootstrap; + + @DataField(id = "outdir", description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) + private String outdir; + + public HRDetectAnalysisParams() { + } + + public HRDetectAnalysisParams(String id, String description, String sampleId, String snvFittingId, String svFittingId, String cnvQuery, + String indelQuery, String snv3CustomName, String snv8CustomName, String sv3CustomName, + String sv8CustomName, Boolean bootstrap, String outdir) { + this.id = id; + this.description = description; + this.sampleId = sampleId; + this.snvFittingId = snvFittingId; + this.svFittingId = svFittingId; + this.cnvQuery = cnvQuery; + this.indelQuery = indelQuery; + this.snv3CustomName = snv3CustomName; + this.snv8CustomName = snv8CustomName; + this.sv3CustomName = sv3CustomName; + this.sv8CustomName = sv8CustomName; + this.bootstrap = bootstrap; + this.outdir = outdir; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("HRDetectAnalysisParams{"); + sb.append("id='").append(id).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", sampleId='").append(sampleId).append('\''); + sb.append(", snvFittingId='").append(snvFittingId).append('\''); + sb.append(", svFittingId='").append(svFittingId).append('\''); + sb.append(", cnvQuery='").append(cnvQuery).append('\''); + sb.append(", indelQuery='").append(indelQuery).append('\''); + sb.append(", snv3CustomName='").append(snv3CustomName).append('\''); + sb.append(", snv8CustomName='").append(snv8CustomName).append('\''); + sb.append(", sv3CustomName='").append(sv3CustomName).append('\''); + sb.append(", sv8CustomName='").append(sv8CustomName).append('\''); + sb.append(", bootstrap=").append(bootstrap); + sb.append(", outdir='").append(outdir).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public HRDetectAnalysisParams setId(String id) { + this.id = id; + return this; + } + + public String getDescription() { + return description; + } + + public HRDetectAnalysisParams setDescription(String description) { + this.description = description; + return this; + } + + public String getSampleId() { + return sampleId; + } + + public HRDetectAnalysisParams setSampleId(String sampleId) { + this.sampleId = sampleId; + return this; + } + + public String getSnvFittingId() { + return snvFittingId; + } + + public HRDetectAnalysisParams setSnvFittingId(String snvFittingId) { + this.snvFittingId = snvFittingId; + return this; + } + + public String getSvFittingId() { + return svFittingId; + } + + public HRDetectAnalysisParams setSvFittingId(String svFittingId) { + this.svFittingId = svFittingId; + return this; + } + + public String getCnvQuery() { + return cnvQuery; + } + + public HRDetectAnalysisParams setCnvQuery(String cnvQuery) { + this.cnvQuery = cnvQuery; + return this; + } + + public String getIndelQuery() { + return indelQuery; + } + + public HRDetectAnalysisParams setIndelQuery(String indelQuery) { + this.indelQuery = indelQuery; + return this; + } + + public String getSnv3CustomName() { + return snv3CustomName; + } + + public HRDetectAnalysisParams setSnv3CustomName(String snv3CustomName) { + this.snv3CustomName = snv3CustomName; + return this; + } + + public String getSnv8CustomName() { + return snv8CustomName; + } + + public HRDetectAnalysisParams setSnv8CustomName(String snv8CustomName) { + this.snv8CustomName = snv8CustomName; + return this; + } + + public String getSv3CustomName() { + return sv3CustomName; + } + + public HRDetectAnalysisParams setSv3CustomName(String sv3CustomName) { + this.sv3CustomName = sv3CustomName; + return this; + } + + public String getSv8CustomName() { + return sv8CustomName; + } + + public HRDetectAnalysisParams setSv8CustomName(String sv8CustomName) { + this.sv8CustomName = sv8CustomName; + return this; + } + + public Boolean isBootstrap() { + return bootstrap; + } + + public HRDetectAnalysisParams setBootstrap(Boolean bootstrap) { + this.bootstrap = bootstrap; + return this; + } + + public String getOutdir() { + return outdir; + } + + public HRDetectAnalysisParams setOutdir(String outdir) { + this.outdir = outdir; + return this; + } +} + diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/MutationalSignatureAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/MutationalSignatureAnalysisParams.java index b46d37bcf59..5687367e4e3 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/MutationalSignatureAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/MutationalSignatureAnalysisParams.java @@ -21,51 +21,62 @@ import org.opencb.opencga.core.tools.ToolParams; public class MutationalSignatureAnalysisParams extends ToolParams { - public static final String DESCRIPTION = "Mutational signature analysis params"; + public static final String SIGNATURE_CATALOGUE_SKIP_VALUE = "catalogue"; + public static final String SIGNATURE_FITTING_SKIP_VALUE = "fitting"; + + public static final String DESCRIPTION = "Mutational signature analysis parameters to index the genome context for that sample, and" + + " to compute both catalogue counts and signature fitting. In order to skip one of them, , use the following keywords: " + + ", " + SIGNATURE_CATALOGUE_SKIP_VALUE + ", " + SIGNATURE_FITTING_SKIP_VALUE + "."; + + // For counts (i.e., catalogue file) @DataField(id = "id", description = FieldConstants.MUTATIONAL_SIGNATURE_ID_DESCRIPTION) private String id; @DataField(id = "description", description = FieldConstants.MUTATIONAL_SIGNATURE_DESCRIPTION_DESCRIPTION) private String description; + @DataField(id = "sample", description = FieldConstants.MUTATIONAL_SIGNATURE_SAMPLE_DESCRIPTION) + private String sample; + @DataField(id = "query", description = FieldConstants.MUTATIONAL_SIGNATURE_QUERY_DESCRIPTION) private String query; // For fitting method - - @DataField(id = "catalogues", description = FieldConstants.MUTATIONAL_SIGNATURE_CATALOGUES_DESCRIPTION) - private String catalogues; - - @DataField(id = "cataloguesContent", description = FieldConstants.MUTATIONAL_SIGNATURE_CATALOGUES_CONTENT_DESCRIPTION) - private String cataloguesContent; + @DataField(id = "fitId", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_ID_DESCRIPTION) + private String fitId; @DataField(id = "fitMethod", defaultValue = "FitMS", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_METHOD_DESCRIPTION) private String fitMethod; - @DataField(id = "nBoot", description = FieldConstants.MUTATIONAL_SIGNATURE_N_BOOT_DESCRIPTION) - private Integer nBoot; + @DataField(id = "fitNBoot", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_N_BOOT_DESCRIPTION) + private Integer fitNBoot; - @DataField(id = "sigVersion", defaultValue = "RefSigv2", description = FieldConstants.MUTATIONAL_SIGNATURE_SIG_VERSION_DESCRIPTION) - private String sigVersion; + @DataField(id = "fitSigVersion", defaultValue = "RefSigv2", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_SIG_VERSION_DESCRIPTION) + private String fitSigVersion; - @DataField(id = "organ", description = FieldConstants.MUTATIONAL_SIGNATURE_ORGAN_DESCRIPTION) - private String organ; + @DataField(id = "fitOrgan", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_ORGAN_DESCRIPTION) + private String fitOrgan; - @DataField(id = "thresholdPerc", defaultValue = "5f", description = FieldConstants.MUTATIONAL_SIGNATURE_THRESHOLD_PERC_DESCRIPTION) - private Float thresholdPerc; + @DataField(id = "fitThresholdPerc", defaultValue = "5f", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_THRESHOLD_PERC_DESCRIPTION) + private Float fitThresholdPerc; - @DataField(id = "thresholdPval", defaultValue = "0.05f", description = FieldConstants.MUTATIONAL_SIGNATURE_THRESHOLD_PVAL_DESCRIPTION) - private Float thresholdPval; + @DataField(id = "fitThresholdPval", defaultValue = "0.05f", + description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_THRESHOLD_PVAL_DESCRIPTION) + private Float fitThresholdPval; - @DataField(id = "maxRareSigs", defaultValue = "1", description = FieldConstants.MUTATIONAL_SIGNATURE_MAX_RARE_SIGS_DESCRIPTION) - private Integer maxRareSigs; + @DataField(id = "fitMaxRareSigs", defaultValue = "1", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_MAX_RARE_SIGS_DESCRIPTION) + private Integer fitMaxRareSigs; - @DataField(id = "signaturesFile", description = FieldConstants.MUTATIONAL_SIGNATURE_SIGNATURES_FILE_DESCRIPTION) - private String signaturesFile; + @DataField(id = "fitSignaturesFile", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_SIGNATURES_FILE_DESCRIPTION) + private String fitSignaturesFile; - @DataField(id = "rareSignaturesFile", description = FieldConstants.MUTATIONAL_SIGNATURE_RARE_SIGNATURES_FILE_DESCRIPTION) - private String rareSignaturesFile; + @DataField(id = "fitRareSignaturesFile", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_RARE_SIGNATURES_FILE_DESCRIPTION) + private String fitRareSignaturesFile; + + // Other + @DataField(id = "skip", description = FieldConstants.MUTATIONAL_SIGNATURE_SKIP_DESCRIPTION) + private String skip; @DataField(id = "outdir", description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) private String outdir; @@ -73,24 +84,25 @@ public class MutationalSignatureAnalysisParams extends ToolParams { public MutationalSignatureAnalysisParams() { } - public MutationalSignatureAnalysisParams(String id, String description, String query, String catalogues, String cataloguesContent, - String fitMethod, Integer nBoot, String sigVersion, String organ, Float thresholdPerc, - Float thresholdPval, Integer maxRareSigs, String signaturesFile, String rareSignaturesFile, - String outdir) { + public MutationalSignatureAnalysisParams(String id, String description, String sample, String query, String fitId, String fitMethod, + Integer fitNBoot, String fitSigVersion, String fitOrgan, Float fitThresholdPerc, + Float fitThresholdPval, Integer fitMaxRareSigs, String fitSignaturesFile, + String fitRareSignaturesFile, String skip, String outdir) { this.id = id; this.description = description; + this.sample = sample; this.query = query; - this.catalogues = catalogues; - this.cataloguesContent = cataloguesContent; + this.fitId = fitId; this.fitMethod = fitMethod; - this.nBoot = nBoot; - this.sigVersion = sigVersion; - this.organ = organ; - this.thresholdPerc = thresholdPerc; - this.thresholdPval = thresholdPval; - this.maxRareSigs = maxRareSigs; - this.signaturesFile = signaturesFile; - this.rareSignaturesFile = rareSignaturesFile; + this.fitNBoot = fitNBoot; + this.fitSigVersion = fitSigVersion; + this.fitOrgan = fitOrgan; + this.fitThresholdPerc = fitThresholdPerc; + this.fitThresholdPval = fitThresholdPval; + this.fitMaxRareSigs = fitMaxRareSigs; + this.fitSignaturesFile = fitSignaturesFile; + this.fitRareSignaturesFile = fitRareSignaturesFile; + this.skip = skip; this.outdir = outdir; } @@ -99,18 +111,19 @@ public String toString() { final StringBuilder sb = new StringBuilder("MutationalSignatureAnalysisParams{"); sb.append("id='").append(id).append('\''); sb.append(", description='").append(description).append('\''); + sb.append(", sample='").append(sample).append('\''); sb.append(", query='").append(query).append('\''); - sb.append(", catalogues='").append(catalogues).append('\''); - sb.append(", cataloguesContent='").append(cataloguesContent).append('\''); + sb.append(", fitId='").append(fitId).append('\''); sb.append(", fitMethod='").append(fitMethod).append('\''); - sb.append(", nBoot=").append(nBoot); - sb.append(", sigVersion='").append(sigVersion).append('\''); - sb.append(", organ='").append(organ).append('\''); - sb.append(", thresholdPerc=").append(thresholdPerc); - sb.append(", thresholdPval=").append(thresholdPval); - sb.append(", maxRareSigs=").append(maxRareSigs); - sb.append(", signaturesFile='").append(signaturesFile).append('\''); - sb.append(", rareSignaturesFile='").append(rareSignaturesFile).append('\''); + sb.append(", fitNBoot=").append(fitNBoot); + sb.append(", fitSigVersion='").append(fitSigVersion).append('\''); + sb.append(", fitOrgan='").append(fitOrgan).append('\''); + sb.append(", fitThresholdPerc=").append(fitThresholdPerc); + sb.append(", fitThresholdPval=").append(fitThresholdPval); + sb.append(", fitMaxRareSigs=").append(fitMaxRareSigs); + sb.append(", fitSignaturesFile='").append(fitSignaturesFile).append('\''); + sb.append(", fitRareSignaturesFile='").append(fitRareSignaturesFile).append('\''); + sb.append(", skip='").append(skip).append('\''); sb.append(", outdir='").append(outdir).append('\''); sb.append('}'); return sb.toString(); @@ -134,30 +147,30 @@ public MutationalSignatureAnalysisParams setDescription(String description) { return this; } - public String getQuery() { - return query; + public String getSample() { + return sample; } - public MutationalSignatureAnalysisParams setQuery(String query) { - this.query = query; + public MutationalSignatureAnalysisParams setSample(String sample) { + this.sample = sample; return this; } - public String getCatalogues() { - return catalogues; + public String getQuery() { + return query; } - public MutationalSignatureAnalysisParams setCatalogues(String catalogues) { - this.catalogues = catalogues; + public MutationalSignatureAnalysisParams setQuery(String query) { + this.query = query; return this; } - public String getCataloguesContent() { - return cataloguesContent; + public String getFitId() { + return fitId; } - public MutationalSignatureAnalysisParams setCataloguesContent(String cataloguesContent) { - this.cataloguesContent = cataloguesContent; + public MutationalSignatureAnalysisParams setFitId(String fitId) { + this.fitId = fitId; return this; } @@ -170,75 +183,84 @@ public MutationalSignatureAnalysisParams setFitMethod(String fitMethod) { return this; } - public Integer getnBoot() { - return nBoot; + public Integer getFitNBoot() { + return fitNBoot; + } + + public MutationalSignatureAnalysisParams setFitNBoot(Integer fitNBoot) { + this.fitNBoot = fitNBoot; + return this; + } + + public String getFitSigVersion() { + return fitSigVersion; } - public MutationalSignatureAnalysisParams setnBoot(Integer nBoot) { - this.nBoot = nBoot; + public MutationalSignatureAnalysisParams setFitSigVersion(String fitSigVersion) { + this.fitSigVersion = fitSigVersion; return this; } - public String getSigVersion() { - return sigVersion; + public String getFitOrgan() { + return fitOrgan; } - public MutationalSignatureAnalysisParams setSigVersion(String sigVersion) { - this.sigVersion = sigVersion; + public MutationalSignatureAnalysisParams setFitOrgan(String fitOrgan) { + this.fitOrgan = fitOrgan; return this; } - public String getOrgan() { - return organ; + public Float getFitThresholdPerc() { + return fitThresholdPerc; } - public MutationalSignatureAnalysisParams setOrgan(String organ) { - this.organ = organ; + public MutationalSignatureAnalysisParams setFitThresholdPerc(Float fitThresholdPerc) { + this.fitThresholdPerc = fitThresholdPerc; return this; } - public Float getThresholdPerc() { - return thresholdPerc; + public Float getFitThresholdPval() { + return fitThresholdPval; } - public MutationalSignatureAnalysisParams setThresholdPerc(Float thresholdPerc) { - this.thresholdPerc = thresholdPerc; + public MutationalSignatureAnalysisParams setFitThresholdPval(Float fitThresholdPval) { + this.fitThresholdPval = fitThresholdPval; return this; } - public Float getThresholdPval() { - return thresholdPval; + public Integer getFitMaxRareSigs() { + return fitMaxRareSigs; } - public MutationalSignatureAnalysisParams setThresholdPval(Float thresholdPval) { - this.thresholdPval = thresholdPval; + public MutationalSignatureAnalysisParams setFitMaxRareSigs(Integer fitMaxRareSigs) { + this.fitMaxRareSigs = fitMaxRareSigs; return this; } - public Integer getMaxRareSigs() { - return maxRareSigs; + public String getFitSignaturesFile() { + return fitSignaturesFile; } - public MutationalSignatureAnalysisParams setMaxRareSigs(Integer maxRareSigs) { - this.maxRareSigs = maxRareSigs; + public MutationalSignatureAnalysisParams setFitSignaturesFile(String fitSignaturesFile) { + this.fitSignaturesFile = fitSignaturesFile; return this; } - public String getSignaturesFile() { - return signaturesFile; + public String getFitRareSignaturesFile() { + return fitRareSignaturesFile; } - public MutationalSignatureAnalysisParams setSignaturesFile(String signaturesFile) { - this.signaturesFile = signaturesFile; + public MutationalSignatureAnalysisParams setFitRareSignaturesFile(String fitRareSignaturesFile) { + this.fitRareSignaturesFile = fitRareSignaturesFile; return this; } - public String getRareSignaturesFile() { - return rareSignaturesFile; + public String getSkip() { + return skip; } - public MutationalSignatureAnalysisParams setRareSignaturesFile(String rareSignaturesFile) { - this.rareSignaturesFile = rareSignaturesFile; + public MutationalSignatureAnalysisParams setSkip(String skip) { + this.skip = skip; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/SampleQcAnalysisParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/SampleQcAnalysisParams.java index 141fc4420ab..6afc8f3049a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/SampleQcAnalysisParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/SampleQcAnalysisParams.java @@ -21,8 +21,18 @@ import org.opencb.opencga.core.tools.ToolParams; public class SampleQcAnalysisParams extends ToolParams { + public static final String VARIANT_STATS_SKIP_VALUE = "variant-stats"; + public static final String SIGNATURE_SKIP_VALUE = "signature"; + public static final String SIGNATURE_CATALOGUE_SKIP_VALUE = "signature-catalogue"; + public static final String SIGNATURE_FITTING_SKIP_VALUE = "signature-fitting"; + public static final String GENOME_PLOT_SKIP_VALUE = "genome-plot"; + public static final String DESCRIPTION = "Sample QC analysis params. Mutational signature and genome plot are calculated for somatic" - + " samples only"; + + " samples only. In order to skip some metrics, use the following keywords (separated by commas): " + VARIANT_STATS_SKIP_VALUE + + ", " + SIGNATURE_SKIP_VALUE + ", " + SIGNATURE_CATALOGUE_SKIP_VALUE + ", " + SIGNATURE_FITTING_SKIP_VALUE + ", " + + GENOME_PLOT_SKIP_VALUE; + + @DataField(id = "sample", description = FieldConstants.SAMPLE_ID_DESCRIPTION) private String sample; // Variant stats params @@ -46,32 +56,38 @@ public class SampleQcAnalysisParams extends ToolParams { @DataField(id = "msQuery", description = FieldConstants.MUTATIONAL_SIGNATURE_QUERY_DESCRIPTION) private String msQuery; + @DataField(id = "msFitId", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_METHOD_DESCRIPTION) + private String msFitId; + @DataField(id = "msFitMethod", defaultValue = "FitMS", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_METHOD_DESCRIPTION) private String msFitMethod; - @DataField(id = "msNBoot", description = FieldConstants.MUTATIONAL_SIGNATURE_N_BOOT_DESCRIPTION) - private Integer msNBoot; + @DataField(id = "msFitNBoot", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_N_BOOT_DESCRIPTION) + private Integer msFitNBoot; - @DataField(id = "msSigVersion", defaultValue = "RefSigv2", description = FieldConstants.MUTATIONAL_SIGNATURE_SIG_VERSION_DESCRIPTION) - private String msSigVersion; + @DataField(id = "msFitSigVersion", defaultValue = "RefSigv2", + description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_SIG_VERSION_DESCRIPTION) + private String msFitSigVersion; - @DataField(id = "msOrgan", description = FieldConstants.MUTATIONAL_SIGNATURE_ORGAN_DESCRIPTION) - private String msOrgan; + @DataField(id = "msFitOrgan", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_ORGAN_DESCRIPTION) + private String msFitOrgan; - @DataField(id = "msThresholdPerc", defaultValue = "5f", description = FieldConstants.MUTATIONAL_SIGNATURE_THRESHOLD_PERC_DESCRIPTION) - private Float msThresholdPerc; + @DataField(id = "msFitThresholdPerc", defaultValue = "5f", + description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_THRESHOLD_PERC_DESCRIPTION) + private Float msFitThresholdPerc; - @DataField(id = "msThresholdPval", defaultValue = "0.05f", description = FieldConstants.MUTATIONAL_SIGNATURE_THRESHOLD_PVAL_DESCRIPTION) - private Float msThresholdPval; + @DataField(id = "msFitThresholdPval", defaultValue = "0.05f", + description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_THRESHOLD_PVAL_DESCRIPTION) + private Float msFitThresholdPval; - @DataField(id = "msMaxRareSigs", defaultValue = "1", description = FieldConstants.MUTATIONAL_SIGNATURE_MAX_RARE_SIGS_DESCRIPTION) - private Integer msMaxRareSigs; + @DataField(id = "msFitMaxRareSigs", defaultValue = "1", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_MAX_RARE_SIGS_DESCRIPTION) + private Integer msFitMaxRareSigs; - @DataField(id = "msSignaturesFile", description = FieldConstants.MUTATIONAL_SIGNATURE_SIGNATURES_FILE_DESCRIPTION) - private String msSignaturesFile; + @DataField(id = "msFitSignaturesFile", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_SIGNATURES_FILE_DESCRIPTION) + private String msFitSignaturesFile; - @DataField(id = "msRareSignaturesFile", description = FieldConstants.MUTATIONAL_SIGNATURE_RARE_SIGNATURES_FILE_DESCRIPTION) - private String msRareSignaturesFile; + @DataField(id = "msFitRareSignaturesFile", description = FieldConstants.MUTATIONAL_SIGNATURE_FIT_RARE_SIGNATURES_FILE_DESCRIPTION) + private String msFitRareSignaturesFile; // Genome plot @@ -84,6 +100,10 @@ public class SampleQcAnalysisParams extends ToolParams { @DataField(id = "gpConfigFile", description = FieldConstants.GENOME_PLOT_CONFIGURATION_FILE_DESCRIPTION) private String gpConfigFile; + // Other + @DataField(id = "skip", description = FieldConstants.SAMPLE_QUALITY_CONTROL_SKIP_DESCRIPTION) + private String skip; + @DataField(id = "outdir", description = FieldConstants.JOB_OUT_DIR_DESCRIPTION) private String outdir; @@ -91,10 +111,10 @@ public SampleQcAnalysisParams() { } public SampleQcAnalysisParams(String sample, String vsId, String vsDescription, AnnotationVariantQueryParams vsQuery, String msId, - String msDescription, String msQuery, String msFitMethod, Integer msNBoot, String msSigVersion, - String msOrgan, Float msThresholdPerc, Float msThresholdPval, Integer msMaxRareSigs, - String msSignaturesFile, String msRareSignaturesFile, String gpId, String gpDescription, - String gpConfigFile, String outdir) { + String msDescription, String msQuery, String msFitId, String msFitMethod, Integer msFitNBoot, + String msFitSigVersion, String msFitOrgan, Float msFitThresholdPerc, Float msFitThresholdPval, + Integer msFitMaxRareSigs, String msFitSignaturesFile, String msFitRareSignaturesFile, String gpId, + String gpDescription, String gpConfigFile, String skip, String outdir) { this.sample = sample; this.vsId = vsId; this.vsDescription = vsDescription; @@ -102,18 +122,20 @@ public SampleQcAnalysisParams(String sample, String vsId, String vsDescription, this.msId = msId; this.msDescription = msDescription; this.msQuery = msQuery; + this.msFitId = msFitId; this.msFitMethod = msFitMethod; - this.msNBoot = msNBoot; - this.msSigVersion = msSigVersion; - this.msOrgan = msOrgan; - this.msThresholdPerc = msThresholdPerc; - this.msThresholdPval = msThresholdPval; - this.msMaxRareSigs = msMaxRareSigs; - this.msSignaturesFile = msSignaturesFile; - this.msRareSignaturesFile = msRareSignaturesFile; + this.msFitNBoot = msFitNBoot; + this.msFitSigVersion = msFitSigVersion; + this.msFitOrgan = msFitOrgan; + this.msFitThresholdPerc = msFitThresholdPerc; + this.msFitThresholdPval = msFitThresholdPval; + this.msFitMaxRareSigs = msFitMaxRareSigs; + this.msFitSignaturesFile = msFitSignaturesFile; + this.msFitRareSignaturesFile = msFitRareSignaturesFile; this.gpId = gpId; this.gpDescription = gpDescription; this.gpConfigFile = gpConfigFile; + this.skip = skip; this.outdir = outdir; } @@ -127,18 +149,20 @@ public String toString() { sb.append(", msId='").append(msId).append('\''); sb.append(", msDescription='").append(msDescription).append('\''); sb.append(", msQuery='").append(msQuery).append('\''); + sb.append(", msFitId='").append(msFitId).append('\''); sb.append(", msFitMethod='").append(msFitMethod).append('\''); - sb.append(", msNBoot=").append(msNBoot); - sb.append(", msSigVersion='").append(msSigVersion).append('\''); - sb.append(", msOrgan='").append(msOrgan).append('\''); - sb.append(", msThresholdPerc=").append(msThresholdPerc); - sb.append(", msThresholdPval=").append(msThresholdPval); - sb.append(", msMaxRareSigs=").append(msMaxRareSigs); - sb.append(", msSignaturesFile='").append(msSignaturesFile).append('\''); - sb.append(", msRareSignaturesFile='").append(msRareSignaturesFile).append('\''); + sb.append(", msFitNBoot=").append(msFitNBoot); + sb.append(", msFitSigVersion='").append(msFitSigVersion).append('\''); + sb.append(", msFitOrgan='").append(msFitOrgan).append('\''); + sb.append(", msFitThresholdPerc=").append(msFitThresholdPerc); + sb.append(", msFitThresholdPval=").append(msFitThresholdPval); + sb.append(", msFitMaxRareSigs=").append(msFitMaxRareSigs); + sb.append(", msFitSignaturesFile='").append(msFitSignaturesFile).append('\''); + sb.append(", msFitRareSignaturesFile='").append(msFitRareSignaturesFile).append('\''); sb.append(", gpId='").append(gpId).append('\''); sb.append(", gpDescription='").append(gpDescription).append('\''); sb.append(", gpConfigFile='").append(gpConfigFile).append('\''); + sb.append(", skip='").append(skip).append('\''); sb.append(", outdir='").append(outdir).append('\''); sb.append('}'); return sb.toString(); @@ -207,6 +231,15 @@ public SampleQcAnalysisParams setMsQuery(String msQuery) { return this; } + public String getMsFitId() { + return msFitId; + } + + public SampleQcAnalysisParams setMsFitId(String msFitId) { + this.msFitId = msFitId; + return this; + } + public String getMsFitMethod() { return msFitMethod; } @@ -216,75 +249,75 @@ public SampleQcAnalysisParams setMsFitMethod(String msFitMethod) { return this; } - public Integer getMsNBoot() { - return msNBoot; + public Integer getMsFitNBoot() { + return msFitNBoot; } - public SampleQcAnalysisParams setMsNBoot(Integer msNBoot) { - this.msNBoot = msNBoot; + public SampleQcAnalysisParams setMsFitNBoot(Integer msFitNBoot) { + this.msFitNBoot = msFitNBoot; return this; } - public String getMsSigVersion() { - return msSigVersion; + public String getMsFitSigVersion() { + return msFitSigVersion; } - public SampleQcAnalysisParams setMsSigVersion(String msSigVersion) { - this.msSigVersion = msSigVersion; + public SampleQcAnalysisParams setMsFitSigVersion(String msFitSigVersion) { + this.msFitSigVersion = msFitSigVersion; return this; } - public String getMsOrgan() { - return msOrgan; + public String getMsFitOrgan() { + return msFitOrgan; } - public SampleQcAnalysisParams setMsOrgan(String msOrgan) { - this.msOrgan = msOrgan; + public SampleQcAnalysisParams setMsFitOrgan(String msFitOrgan) { + this.msFitOrgan = msFitOrgan; return this; } - public Float getMsThresholdPerc() { - return msThresholdPerc; + public Float getMsFitThresholdPerc() { + return msFitThresholdPerc; } - public SampleQcAnalysisParams setMsThresholdPerc(Float msThresholdPerc) { - this.msThresholdPerc = msThresholdPerc; + public SampleQcAnalysisParams setMsFitThresholdPerc(Float msFitThresholdPerc) { + this.msFitThresholdPerc = msFitThresholdPerc; return this; } - public Float getMsThresholdPval() { - return msThresholdPval; + public Float getMsFitThresholdPval() { + return msFitThresholdPval; } - public SampleQcAnalysisParams setMsThresholdPval(Float msThresholdPval) { - this.msThresholdPval = msThresholdPval; + public SampleQcAnalysisParams setMsFitThresholdPval(Float msFitThresholdPval) { + this.msFitThresholdPval = msFitThresholdPval; return this; } - public Integer getMsMaxRareSigs() { - return msMaxRareSigs; + public Integer getMsFitMaxRareSigs() { + return msFitMaxRareSigs; } - public SampleQcAnalysisParams setMsMaxRareSigs(Integer msMaxRareSigs) { - this.msMaxRareSigs = msMaxRareSigs; + public SampleQcAnalysisParams setMsFitMaxRareSigs(Integer msFitMaxRareSigs) { + this.msFitMaxRareSigs = msFitMaxRareSigs; return this; } - public String getMsSignaturesFile() { - return msSignaturesFile; + public String getMsFitSignaturesFile() { + return msFitSignaturesFile; } - public SampleQcAnalysisParams setMsSignaturesFile(String msSignaturesFile) { - this.msSignaturesFile = msSignaturesFile; + public SampleQcAnalysisParams setMsFitSignaturesFile(String msFitSignaturesFile) { + this.msFitSignaturesFile = msFitSignaturesFile; return this; } - public String getMsRareSignaturesFile() { - return msRareSignaturesFile; + public String getMsFitRareSignaturesFile() { + return msFitRareSignaturesFile; } - public SampleQcAnalysisParams setMsRareSignaturesFile(String msRareSignaturesFile) { - this.msRareSignaturesFile = msRareSignaturesFile; + public SampleQcAnalysisParams setMsFitRareSignaturesFile(String msFitRareSignaturesFile) { + this.msFitRareSignaturesFile = msFitRareSignaturesFile; return this; } @@ -315,6 +348,15 @@ public SampleQcAnalysisParams setGpConfigFile(String gpConfigFile) { return this; } + public String getSkip() { + return skip; + } + + public SampleQcAnalysisParams setSkip(String skip) { + this.skip = skip; + return this; + } + public String getOutdir() { return outdir; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/HRDetectAnalysisExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/HRDetectAnalysisExecutor.java new file mode 100644 index 00000000000..afaa4f966a7 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/HRDetectAnalysisExecutor.java @@ -0,0 +1,197 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.core.tools.variant; + +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.core.tools.OpenCgaToolExecutor; + +import java.nio.file.Path; + +public abstract class HRDetectAnalysisExecutor extends OpenCgaToolExecutor { + + private String study; + private String somaticSample; + private String germlineSample; + private String assembly; + private Path snvRDataPath; + private Path svRDataPath; + private ObjectMap cnvQuery; + private ObjectMap indelQuery; + private String snv3CustomName; + private String snv8CustomName; + private String sv3CustomName; + private String sv8CustomName; + private Boolean bootstrap; + + public HRDetectAnalysisExecutor() { + } + + public HRDetectAnalysisExecutor(String study, String somaticSample, String germlineSample, String assembly, Path snvRDataPath, + Path svRDataPath, ObjectMap cnvQuery, ObjectMap indelQuery, String snv3CustomName, + String snv8CustomName, String sv3CustomName, String sv8CustomName, Boolean bootstrap) { + this.study = study; + this.somaticSample = somaticSample; + this.germlineSample = germlineSample; + this.assembly = assembly; + this.snvRDataPath = snvRDataPath; + this.svRDataPath = svRDataPath; + this.cnvQuery = cnvQuery; + this.indelQuery = indelQuery; + this.snv3CustomName = snv3CustomName; + this.snv8CustomName = snv8CustomName; + this.sv3CustomName = sv3CustomName; + this.sv8CustomName = sv8CustomName; + this.bootstrap = bootstrap; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("HRDetectAnalysisExecutor{"); + sb.append("study='").append(study).append('\''); + sb.append(", somaticSample='").append(somaticSample).append('\''); + sb.append(", germlineSample='").append(germlineSample).append('\''); + sb.append(", assembly='").append(assembly).append('\''); + sb.append(", snvRDataPath=").append(snvRDataPath); + sb.append(", svRDataPath=").append(svRDataPath); + sb.append(", cnvQuery=").append(cnvQuery); + sb.append(", indelQuery=").append(indelQuery); + sb.append(", snv3CustomName='").append(snv3CustomName).append('\''); + sb.append(", snv8CustomName='").append(snv8CustomName).append('\''); + sb.append(", sv3CustomName='").append(sv3CustomName).append('\''); + sb.append(", sv8CustomName='").append(sv8CustomName).append('\''); + sb.append(", bootstrap=").append(bootstrap); + sb.append('}'); + return sb.toString(); + } + + public String getStudy() { + return study; + } + + public HRDetectAnalysisExecutor setStudy(String study) { + this.study = study; + return this; + } + + public String getSomaticSample() { + return somaticSample; + } + + public HRDetectAnalysisExecutor setSomaticSample(String somaticSample) { + this.somaticSample = somaticSample; + return this; + } + + public String getGermlineSample() { + return germlineSample; + } + + public HRDetectAnalysisExecutor setGermlineSample(String germlineSample) { + this.germlineSample = germlineSample; + return this; + } + + public String getAssembly() { + return assembly; + } + + public HRDetectAnalysisExecutor setAssembly(String assembly) { + this.assembly = assembly; + return this; + } + + public Path getSnvRDataPath() { + return snvRDataPath; + } + + public HRDetectAnalysisExecutor setSnvRDataPath(Path snvRDataPath) { + this.snvRDataPath = snvRDataPath; + return this; + } + + public Path getSvRDataPath() { + return svRDataPath; + } + + public HRDetectAnalysisExecutor setSvRDataPath(Path svRDataPath) { + this.svRDataPath = svRDataPath; + return this; + } + + public ObjectMap getCnvQuery() { + return cnvQuery; + } + + public HRDetectAnalysisExecutor setCnvQuery(ObjectMap cnvQuery) { + this.cnvQuery = cnvQuery; + return this; + } + + public ObjectMap getIndelQuery() { + return indelQuery; + } + + public HRDetectAnalysisExecutor setIndelQuery(ObjectMap indelQuery) { + this.indelQuery = indelQuery; + return this; + } + + public String getSnv3CustomName() { + return snv3CustomName; + } + + public HRDetectAnalysisExecutor setSnv3CustomName(String snv3CustomName) { + this.snv3CustomName = snv3CustomName; + return this; + } + + public String getSnv8CustomName() { + return snv8CustomName; + } + + public HRDetectAnalysisExecutor setSnv8CustomName(String snv8CustomName) { + this.snv8CustomName = snv8CustomName; + return this; + } + + public String getSv3CustomName() { + return sv3CustomName; + } + + public HRDetectAnalysisExecutor setSv3CustomName(String sv3CustomName) { + this.sv3CustomName = sv3CustomName; + return this; + } + + public String getSv8CustomName() { + return sv8CustomName; + } + + public HRDetectAnalysisExecutor setSv8CustomName(String sv8CustomName) { + this.sv8CustomName = sv8CustomName; + return this; + } + + public Boolean getBootstrap() { + return bootstrap; + } + + public HRDetectAnalysisExecutor setBootstrap(Boolean bootstrap) { + this.bootstrap = bootstrap; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/MutationalSignatureAnalysisExecutor.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/MutationalSignatureAnalysisExecutor.java index 3eb4c68ab37..be8fdcf1a62 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/MutationalSignatureAnalysisExecutor.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/variant/MutationalSignatureAnalysisExecutor.java @@ -41,7 +41,7 @@ public abstract class MutationalSignatureAnalysisExecutor extends OpenCgaToolExe private ObjectMap query; // For fitting signature - private String catalogues; + private String fitId; private String fitMethod; private Integer nBoot; private String sigVersion; @@ -52,15 +52,9 @@ public abstract class MutationalSignatureAnalysisExecutor extends OpenCgaToolExe private String signaturesFile; private String rareSignaturesFile; - public MutationalSignatureAnalysisExecutor() { - } + private String skip; - protected String getContextIndexFilename() { - return "OPENCGA_" + sample + "_" + assembly + "_genome_context.csv"; - } - - protected String getMutationalSignatureFilename() { - return "result_" + sigVersion + "_" + assembly + ".txt"; + public MutationalSignatureAnalysisExecutor() { } protected static Map> initCountMap() { @@ -182,6 +176,15 @@ public MutationalSignatureAnalysisExecutor setAssembly(String assembly) { return this; } + public String getFitId() { + return fitId; + } + + public MutationalSignatureAnalysisExecutor setFitId(String fitId) { + this.fitId = fitId; + return this; + } + public String getFitMethod() { return fitMethod; } @@ -213,15 +216,6 @@ public MutationalSignatureAnalysisExecutor setQuery(ObjectMap query) { return this; } - public String getCatalogues() { - return catalogues; - } - - public MutationalSignatureAnalysisExecutor setCatalogues(String catalogues) { - this.catalogues = catalogues; - return this; - } - public MutationalSignatureAnalysisExecutor setFitMethod(String fitMethod) { this.fitMethod = fitMethod; return this; @@ -298,4 +292,13 @@ public MutationalSignatureAnalysisExecutor setRareSignaturesFile(String rareSign this.rareSignaturesFile = rareSignaturesFile; return this; } + + public String getSkip() { + return skip; + } + + public MutationalSignatureAnalysisExecutor setSkip(String skip) { + this.skip = skip; + return this; + } } diff --git a/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java b/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java index a92b1a7876b..7a87a63ec27 100644 --- a/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java +++ b/opencga-core/src/test/java/org/opencb/opencga/core/config/ConfigurationTest.java @@ -98,21 +98,18 @@ public void testDefault() { @Test public void testLoad() throws Exception { - URL url = new URL("http://resources.opencb.org/opencb/opencga/disease-panels/sources.txt"); - BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(), Charset.defaultCharset())); - - Set sources = new HashSet<>(); - String line; - while((line = reader.readLine()) != null) { - sources.add(line); - } - System.out.println(sources); - - - - - File file = new File(url.toURI()); - System.out.println(file.list()); +// URL url = new URL("http://resources.opencb.org/opencb/opencga/disease-panels/sources.txt"); +// BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream(), Charset.defaultCharset())); +// +// Set sources = new HashSet<>(); +// String line; +// while((line = reader.readLine()) != null) { +// sources.add(line); +// } +// System.out.println(sources); +// +// File file = new File(url.toURI()); +// System.out.println(file.list()); Configuration configuration = Configuration .load(getClass().getResource("/configuration-test.yml").openStream()); diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index ec82033bd33..733961b36b8 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -22,7 +22,7 @@ opencga org.opencb.opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java index 8bfa12404d0..d1562f69d0c 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java @@ -51,6 +51,7 @@ import org.opencb.opencga.analysis.tools.ToolFactory; import org.opencb.opencga.analysis.variant.VariantExportTool; import org.opencb.opencga.analysis.variant.gwas.GwasAnalysis; +import org.opencb.opencga.analysis.variant.hrdetect.HRDetectAnalysis; import org.opencb.opencga.analysis.variant.inferredSex.InferredSexAnalysis; import org.opencb.opencga.analysis.variant.julie.JulieTool; import org.opencb.opencga.analysis.variant.knockout.KnockoutAnalysis; @@ -229,6 +230,7 @@ public class ExecutionDaemon extends MonitorParentDaemon { put(KnockoutAnalysis.ID, "variant knockout-run"); put(SampleEligibilityAnalysis.ID, "variant " + SampleEligibilityAnalysis.ID + "-run"); put(MutationalSignatureAnalysis.ID, "variant " + MutationalSignatureAnalysis.ID + "-run"); + put(HRDetectAnalysis.ID, "variant " + HRDetectAnalysis.ID + "-run"); put(MendelianErrorAnalysis.ID, "variant " + MendelianErrorAnalysis.ID + "-run"); put(InferredSexAnalysis.ID, "variant " + InferredSexAnalysis.ID + "-run"); put(RelatednessAnalysis.ID, "variant " + RelatednessAnalysis.ID + "-run"); diff --git a/opencga-server/pom.xml b/opencga-server/pom.xml index c5faeae1c83..4443a23bcf5 100644 --- a/opencga-server/pom.xml +++ b/opencga-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java index 43fd48f9f7c..185064c6294 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/admin/AdminWSServer.java @@ -267,7 +267,7 @@ public Response install( @ApiParam(value = "JSON containing the mandatory parameters", required = true) InstallationParams installParams) { try { catalogManager.installCatalogDB(installParams.getSecretKey(), installParams.getPassword(), installParams.getEmail(), - installParams.getOrganization(), false, true); + installParams.getOrganization(), false); return createOkResponse(DataResult.empty()); } catch (Exception e) { return createErrorResponse(e); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java index 99c9fabd46e..6ac26bbb92e 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/analysis/VariantWebService.java @@ -39,6 +39,7 @@ import org.opencb.opencga.analysis.variant.circos.CircosLocalAnalysisExecutor; import org.opencb.opencga.analysis.variant.genomePlot.GenomePlotAnalysis; import org.opencb.opencga.analysis.variant.gwas.GwasAnalysis; +import org.opencb.opencga.analysis.variant.hrdetect.HRDetectAnalysis; import org.opencb.opencga.analysis.variant.inferredSex.InferredSexAnalysis; import org.opencb.opencga.analysis.variant.knockout.KnockoutAnalysis; import org.opencb.opencga.analysis.variant.knockout.KnockoutAnalysisResultReader; @@ -61,10 +62,12 @@ import org.opencb.opencga.analysis.wrappers.rvtests.RvtestsWrapperAnalysis; import org.opencb.opencga.analysis.wrappers.samtools.SamtoolsWrapperAnalysis; import org.opencb.opencga.catalog.db.api.FileDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.AvroToAnnotationConverter; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.FieldConstants; import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.exceptions.VersionException; @@ -100,6 +103,7 @@ import java.util.*; import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.SAVED_FILTER_DESCR; +import static org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils.geneRegionIntersect; import static org.opencb.opencga.core.api.ParamConstants.JOB_DEPENDS_ON; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; import static org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam.*; @@ -943,6 +947,7 @@ public Response mutationalSignatureRun( @ApiImplicitParams({ @ApiImplicitParam(name = "study", value = STUDY_DESCR, dataType = "string", paramType = "query"), @ApiImplicitParam(name = "sample", value = "Sample name", dataType = "string", paramType = "query"), + @ApiImplicitParam(name = "type", value = "Variant type. Valid values: SNV, SV", dataType = "string", paramType = "query"), @ApiImplicitParam(name = "ct", value = ANNOT_CONSEQUENCE_TYPE_DESCR, dataType = "string", paramType = "query"), @ApiImplicitParam(name = "biotype", value = ANNOT_BIOTYPE_DESCR, dataType = "string", paramType = "query"), @ApiImplicitParam(name = "fileData", value = FILE_DATA_DESCR, dataType = "string", paramType = "query"), @@ -958,59 +963,52 @@ public Response mutationalSignatureRun( @ApiImplicitParam(name = "panelIntersection", value = VariantCatalogQueryUtils.PANEL_INTERSECTION_DESC, dataType = "boolean", paramType = "query"), }) public Response mutationalSignatureQuery( - // For fitting method - @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_CATALOGUES_DESCRIPTION) @QueryParam("catalogues") String catalogues, - @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_CATALOGUES_CONTENT_DESCRIPTION) @QueryParam("cataloguesContent") String cataloguesContent, - @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_FIT_METHOD_DESCRIPTION, defaultValue = "FitMS") @QueryParam("fitMethod") String fitMethod, - @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_N_BOOT_DESCRIPTION) @QueryParam("nBoot") Integer nBoot, - @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_SIG_VERSION_DESCRIPTION, defaultValue = "RefSigv2") @QueryParam("sigVersion") String sigVersion, - @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_ORGAN_DESCRIPTION) @QueryParam("organ") String organ, - @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_THRESHOLD_PERC_DESCRIPTION, defaultValue = "5f") @QueryParam("thresholdPerc") Float thresholdPerc, - @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_THRESHOLD_PVAL_DESCRIPTION, defaultValue = "0.05f") @QueryParam("thresholdPval") Float thresholdPval, - @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_MAX_RARE_SIGS_DESCRIPTION, defaultValue = "1") @QueryParam("maxRareSigs") Integer maxRareSigs, - @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_SIGNATURES_FILE_DESCRIPTION) @QueryParam("signaturesFile") String signaturesFile, - @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_RARE_SIGNATURES_FILE_DESCRIPTION) @QueryParam("rareSignaturesFile") String rareSignaturesFile + @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_ID_DESCRIPTION) @QueryParam("msId") String msId, + @ApiParam(value = FieldConstants.MUTATIONAL_SIGNATURE_DESCRIPTION_DESCRIPTION) @QueryParam("msDescription") String msDescription ) { - File outDir = null; + java.nio.file.Path outDir = null; try { QueryOptions queryOptions = new QueryOptions(uriInfo.getQueryParameters(), true); Query query = getVariantQuery(queryOptions); + if (!query.containsKey(STUDY.key())) { + return createErrorResponse(new Exception("Missing study name")); + } + if (!query.containsKey(SAMPLE.key())) { return createErrorResponse(new Exception("Missing sample name")); } - if (!query.containsKey(STUDY.key())) { - return createErrorResponse(new Exception("Missing study name")); + // Check for genome context index + File genomeContextFile = MutationalSignatureAnalysis.getGenomeContextFile(query.getString(SAMPLE.key()), + query.getString(STUDY.key()), catalogManager, token); + if (genomeContextFile == null || !genomeContextFile.exists()) { + return createErrorResponse(new Exception("Build the genome context file for sample " + query.getString(SAMPLE.key()) + + " before running mutational signature queries. To create the genome context file you can use the command" + + " mutational-signature-run.")); } // Create temporal directory - outDir = Paths.get(configuration.getAnalysis().getScratchDir(), "mutational-signature-" + TimeUtils.getTimeMillis()).toFile(); + outDir = Paths.get(configuration.getAnalysis().getScratchDir(), "mutational-signature-" + TimeUtils.getTimeMillis()); try { - FileUtils.forceMkdir(outDir); + FileUtils.forceMkdir(outDir.toFile()); + Runtime.getRuntime().exec("chmod 777 " + outDir.toAbsolutePath()); } catch (IOException e) { throw new IOException("Error creating temporal directory for mutational-signature/query analysis. " + e.getMessage(), e); } MutationalSignatureAnalysisParams params = new MutationalSignatureAnalysisParams(); - params.setQuery(query.toJson()) - .setCatalogues(catalogues) - .setCataloguesContent(cataloguesContent) - .setFitMethod(fitMethod) - .setSigVersion(sigVersion) - .setOrgan(organ) - .setnBoot(nBoot) - .setThresholdPerc(thresholdPerc) - .setThresholdPval(thresholdPval) - .setMaxRareSigs(maxRareSigs) - .setSignaturesFile(signaturesFile) - .setRareSignaturesFile(rareSignaturesFile); + params.setId(msId) + .setDescription(msDescription) + .setQuery(query.toJson()) + .setSample(query.getString(SAMPLE.key())) + .setSkip(MutationalSignatureAnalysisParams.SIGNATURE_FITTING_SKIP_VALUE); logger.info("MutationalSignatureAnalysisParams: {}", params); MutationalSignatureAnalysis mutationalSignatureAnalysis = new MutationalSignatureAnalysis(); - mutationalSignatureAnalysis.setUp(opencgaHome.toString(), catalogManager, storageEngineFactory, new ObjectMap(), - outDir.toPath(), null, token); + mutationalSignatureAnalysis.setUp(opencgaHome.toString(), catalogManager, storageEngineFactory, new ObjectMap(), outDir, null, + token); mutationalSignatureAnalysis.setStudy(query.getString(STUDY.key())); mutationalSignatureAnalysis.setSignatureParams(params); @@ -1018,27 +1016,46 @@ public Response mutationalSignatureQuery( mutationalSignatureAnalysis.start(); watch.stop(); - Signature signature = mutationalSignatureAnalysis.parse(outDir.toPath()); - - OpenCGAResult result = new OpenCGAResult<>(((int) watch.getTime()), Collections.emptyList(), 1, - Collections.singletonList(signature), 1); - return createOkResponse(result); - } catch (ToolException | IOException e) { + logger.info("Parsing mutational signature catalogue results from {}", outDir); + File signatureFile = outDir.resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_DATA_MODEL_FILENAME).toFile(); + if (outDir.resolve(MutationalSignatureAnalysis.MUTATIONAL_SIGNATURE_DATA_MODEL_FILENAME).toFile().exists()) { + Signature signature = JacksonUtils.getDefaultObjectMapper().readerFor(Signature.class).readValue(signatureFile); + OpenCGAResult result = new OpenCGAResult<>(((int) watch.getTime()), Collections.emptyList(), 1, + Collections.singletonList(signature), 1); + return createOkResponse(result); + } else { + return createErrorResponse(new ToolException("Something wrong happened: it could not find the signature output file")); + } + } catch (ToolException | IOException | CatalogException e) { return createErrorResponse(e); } finally { if (outDir != null) { // Delete temporal directory try { - if (outDir.exists()) { - FileUtils.deleteDirectory(outDir); + if (outDir.toFile().exists()) { + logger.info("Deleting scratch directory {}", outDir); + FileUtils.deleteDirectory(outDir.toFile()); } } catch (IOException e) { - logger.warn("Error cleaning scratch directory " + outDir, e); + logger.warn("Error cleaning scratch directory {}", outDir, e); } } } } + @POST + @Path("/hrDetect/run") + @ApiOperation(value = HRDetectAnalysis.DESCRIPTION, response = Job.class) + public Response hrDetectRun( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, + @ApiParam(value = ParamConstants.JOB_ID_CREATION_DESCRIPTION) @QueryParam(ParamConstants.JOB_ID) String jobName, + @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, + @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, + @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = HRDetectAnalysisParams.DESCRIPTION, required = true) HRDetectAnalysisParams params) { + return submitJob(HRDetectAnalysis.ID, study, params, jobName, jobDescription, dependsOn, jobTags); + } + @POST @Path("/mendelianError/run") @ApiOperation(value = MendelianErrorAnalysis.DESCRIPTION, response = Job.class) diff --git a/opencga-storage/opencga-storage-app/pom.xml b/opencga-storage/opencga-storage-app/pom.xml index 0cfc500cb3f..338f5cf8000 100644 --- a/opencga-storage/opencga-storage-app/pom.xml +++ b/opencga-storage/opencga-storage-app/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-benchmark/pom.xml b/opencga-storage/opencga-storage-benchmark/pom.xml index 46ee21326f8..bae6376a163 100644 --- a/opencga-storage/opencga-storage-benchmark/pom.xml +++ b/opencga-storage/opencga-storage-benchmark/pom.xml @@ -22,7 +22,7 @@ opencga-storage org.opencb.opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-core/pom.xml b/opencga-storage/opencga-storage-core/pom.xml index 0e9cc094ed1..f14eddf84ea 100644 --- a/opencga-storage/opencga-storage-core/pom.xml +++ b/opencga-storage/opencga-storage-core/pom.xml @@ -25,7 +25,7 @@ org.opencb.opencga opencga-storage - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java index 355002a1a9b..e192dcf365e 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/VariantStorageMetadataManager.java @@ -56,7 +56,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.opencb.opencga.storage.core.variant.annotation.annotators.AbstractCellBaseVariantAnnotator.toCellBaseSpeciesName; +import static org.opencb.opencga.storage.core.utils.CellBaseUtils.toCellBaseSpeciesName; import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.isNegated; import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.removeNegation; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java index f9e6332912b..8284394b571 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/ProjectMetadata.java @@ -2,6 +2,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; +import org.opencb.cellbase.core.models.DataRelease; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryException; @@ -16,6 +17,7 @@ public class ProjectMetadata { private String species; private String assembly; + private String dataRelease; private int release; @@ -76,18 +78,20 @@ public static class VariantAnnotationMetadata { private Date creationDate; private VariantAnnotatorProgram annotator; private List sourceVersion; + private DataRelease dataRelease; public VariantAnnotationMetadata() { sourceVersion = new ArrayList<>(); } public VariantAnnotationMetadata(int id, String name, Date creationDate, VariantAnnotatorProgram annotator, - List sourceVersion) { + List sourceVersion, DataRelease dataRelease) { this.id = id; this.name = name; this.creationDate = creationDate; this.annotator = annotator; this.sourceVersion = sourceVersion != null ? sourceVersion : new ArrayList<>(); + this.dataRelease = dataRelease; } public int getId() { @@ -134,6 +138,15 @@ public VariantAnnotationMetadata setSourceVersion(List sourceVersion) this.sourceVersion = sourceVersion; return this; } + + public DataRelease getDataRelease() { + return dataRelease; + } + + public VariantAnnotationMetadata setDataRelease(DataRelease dataRelease) { + this.dataRelease = dataRelease; + return this; + } } public static class VariantAnnotatorProgram { @@ -216,13 +229,14 @@ public ProjectMetadata() { } public ProjectMetadata(String species, String assembly, int release) { - this(species, assembly, release, null, null, null); + this(species, assembly, null, release, null, null, null); } - public ProjectMetadata(String species, String assembly, int release, ObjectMap attributes, Map counters, - VariantAnnotationSets annotation) { + public ProjectMetadata(String species, String assembly, String dataRelease, int release, ObjectMap attributes, + Map counters, VariantAnnotationSets annotation) { this.species = species; this.assembly = assembly; + this.dataRelease = dataRelease; this.release = release; this.attributes = attributes != null ? attributes : new ObjectMap(); this.annotation = annotation != null ? annotation : new VariantAnnotationSets(); @@ -230,7 +244,7 @@ public ProjectMetadata(String species, String assembly, int release, ObjectMap a } public ProjectMetadata copy() { - return new ProjectMetadata(species, assembly, release, new ObjectMap(attributes), new HashMap<>(counters), annotation); + return new ProjectMetadata(species, assembly, dataRelease, release, new ObjectMap(attributes), new HashMap<>(counters), annotation); } public String getSpecies() { @@ -251,6 +265,15 @@ public ProjectMetadata setAssembly(String assembly) { return this; } + public String getDataRelease() { + return dataRelease; + } + + public ProjectMetadata setDataRelease(String dataRelease) { + this.dataRelease = dataRelease; + return this; + } + public int getRelease() { return release; } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/utils/CellBaseUtils.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/utils/CellBaseUtils.java index 87d3fa3fafb..c98c7cf92f5 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/utils/CellBaseUtils.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/utils/CellBaseUtils.java @@ -16,6 +16,7 @@ package org.opencb.opencga.storage.core.utils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; import org.opencb.biodata.models.core.Gene; import org.opencb.biodata.models.core.Region; @@ -25,6 +26,7 @@ import org.opencb.cellbase.client.rest.CellBaseClient; import org.opencb.cellbase.client.rest.ParentRestClient; import org.opencb.cellbase.core.ParamConstants; +import org.opencb.cellbase.core.config.SpeciesProperties; import org.opencb.cellbase.core.result.CellBaseDataResponse; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.commons.datastore.core.Query; @@ -95,9 +97,17 @@ public String toString() { } } - public CellBaseUtils(CellBaseClient cellBaseClient, String assembly) { + public CellBaseUtils(CellBaseClient cellBaseClient) { this.cellBaseClient = cellBaseClient; - this.assembly = assembly; + this.assembly = cellBaseClient.getAssembly(); + } + + public static String toCellBaseSpeciesName(String scientificName) { + if (scientificName != null && scientificName.contains(" ")) { + String[] split = scientificName.split(" ", 2); + scientificName = (split[0].charAt(0) + split[1]).toLowerCase(); + } + return scientificName; } public Region getGeneRegion(String geneStr) { @@ -363,4 +373,96 @@ public String getAssembly() { public String getSpecies() { return cellBaseClient.getSpecies(); } + + public String getDataRelease() { + return cellBaseClient.getDataRelease(); + } + + public String getURL() { + return cellBaseClient.getClientConfiguration().getRest().getHosts().get(0); + } + + public String getVersion() { + return cellBaseClient.getClientConfiguration().getVersion(); + } + + public void validateCellBaseConnection() throws IOException { + CellBaseDataResponse species = cellBaseClient.getMetaClient().species(); + if (species == null || species.firstResult() == null) { + throw new IllegalArgumentException("Unable to access cellbase url '" + getURL() + "', version '" + getVersion() + "'"); + } + String serverVersion = getVersionFromServerMajorMinor(); + if (!supportsDataRelease(serverVersion)) { + logger.warn("DataRelease not supported on version '" + serverVersion + ".x'"); + } else { + if (getDataRelease() == null) { + throw new IllegalArgumentException("Missing DataRelease for cellbase " + + "url: '" + getURL() + "'" + + ", version: '" + getVersion() + + "', species: '" + getSpecies() + + "', assembly: '" + getAssembly() + "'"); + } + boolean dataReleaseExists = cellBaseClient.getMetaClient().dataReleases() + .allResults() + .stream() + .anyMatch(dataRelease -> { + return getDataRelease().equals(String.valueOf(dataRelease.getRelease())); + }); + if (!dataReleaseExists) { + throw new IllegalArgumentException("DataRelease '" + getDataRelease() + "' not found on cellbase " + + "url: '" + getURL() + "'" + + ", version: '" + getVersion() + + "', species: '" + getSpecies() + + "', assembly: '" + getAssembly() + "'"); + } + } + } + + public boolean supportsDataRelease() throws IOException { + return supportsDataRelease(getVersionFromServer()); + } + + public static boolean supportsDataRelease(String serverVersion) { + return !majorMinor(serverVersion).equals("5.0") && !major(serverVersion).equals("4"); + } + + public String getVersionFromServerMajor() throws IOException { + return major(getVersionFromServer()); + } + + public String getVersionFromServerMajorMinor() throws IOException { + String serverVersion = getVersionFromServer(); + serverVersion = majorMinor(serverVersion); + return serverVersion; + } + + private static String major(String version) { + return version.split("\\.")[0]; + } + + private static String majorMinor(String version) { + String[] split = version.split("\\."); + if (split.length > 1) { + version = split[0] + "." + split[1]; + } + return version; + } + + public String getVersionFromServer() throws IOException { + String serverVersion = cellBaseClient.getMetaClient().about().firstResult().getString("Version"); + if (StringUtils.isEmpty(serverVersion)) { + serverVersion = cellBaseClient.getMetaClient().about().firstResult().getString("Version: "); + } + return serverVersion; + } + + @Override + public String toString() { + return "URL: '" + getURL() + "', " + + "version '" + getVersion() + "', " + + "species '" + getSpecies() + "', " + + "assembly '" + getAssembly() + "', " + + "dataRelease '" + getDataRelease() + "'"; + + } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index d01074fb401..9040a259f1c 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -82,7 +82,7 @@ import java.util.stream.Collectors; import static org.opencb.opencga.storage.core.variant.VariantStorageOptions.*; -import static org.opencb.opencga.storage.core.variant.annotation.annotators.AbstractCellBaseVariantAnnotator.toCellBaseSpeciesName; +import static org.opencb.opencga.storage.core.utils.CellBaseUtils.toCellBaseSpeciesName; import static org.opencb.opencga.storage.core.variant.query.VariantQueryUtils.*; import static org.opencb.opencga.storage.core.variant.search.VariantSearchUtils.buildSamplesIndexCollectionName; @@ -1041,7 +1041,8 @@ public CellBaseUtils getCellBaseUtils() throws StorageEngineException { species = clientConfiguration.getDefaultSpecies(); } species = toCellBaseSpeciesName(species); - cellBaseUtils = new CellBaseUtils(new CellBaseClient(species, assembly, clientConfiguration), assembly); + cellBaseUtils = new CellBaseUtils(new CellBaseClient(species, assembly, configuration.getCellbase().getDataRelease(), + clientConfiguration)); } return cellBaseUtils; } @@ -1083,7 +1084,8 @@ public VariantSearchManager getVariantSearchManager() throws StorageEngineExcept synchronized (variantSearchManager) { if (variantSearchManager.get() == null) { // TODO One day we should use reflection here reading from storage-configuration.yml - variantSearchManager.set(new VariantSearchManager(getMetadataManager(), configuration, getOptions())); + variantSearchManager.set( + new VariantSearchManager(getMetadataManager(), getCellBaseUtils(), configuration, getOptions())); } } } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantField.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantField.java index fc8d60b4a20..d0eb0a335e6 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantField.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/VariantField.java @@ -60,6 +60,7 @@ public enum VariantField { ANNOTATION_ANCESTRAL_ALLELE(ANNOTATION, "annotation.ancestralAllele"), ANNOTATION_ID(ANNOTATION, "annotation.id"), ANNOTATION_XREFS(ANNOTATION, "annotation.xrefs"), + ANNOTATION_GWAS(ANNOTATION, "annotation.gwas"), ANNOTATION_HGVS(ANNOTATION, "annotation.hgvs"), ANNOTATION_CYTOBAND(ANNOTATION, "annotation.cytoband"), ANNOTATION_DISPLAY_CONSEQUENCE_TYPE(ANNOTATION, "annotation.displayConsequenceType"), diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/DefaultVariantAnnotationManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/DefaultVariantAnnotationManager.java index 86d6a483b91..71e4d827227 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/DefaultVariantAnnotationManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/DefaultVariantAnnotationManager.java @@ -132,11 +132,10 @@ public long annotate(Query query, ObjectMap params) throws VariantAnnotatorExcep preAnnotate(query, doCreate, doLoad, params); if (doCreate && doLoad) { - ProjectMetadata.VariantAnnotatorProgram newAnnotator = variantAnnotator.getVariantAnnotatorProgram(); - List newSourceVersion = variantAnnotator.getVariantAnnotatorSourceVersion(); + ProjectMetadata.VariantAnnotationMetadata newVariantAnnotationMetadata = variantAnnotator.getVariantAnnotationMetadata(); dbAdaptor.getMetadataManager().updateProjectMetadata(projectMetadata -> { - checkCurrentAnnotation(projectMetadata, overwrite, newAnnotator, newSourceVersion); + checkCurrentAnnotation(projectMetadata, overwrite, newVariantAnnotationMetadata); return projectMetadata; }); } @@ -503,12 +502,11 @@ protected void postAnnotate(Query query, boolean doCreate, boolean doLoad, Objec throws VariantAnnotatorException, StorageEngineException, IOException { boolean overwrite = params.getBoolean(VariantStorageOptions.ANNOTATION_OVERWEITE.key(), false); if (doLoad && doCreate) { - ProjectMetadata.VariantAnnotatorProgram newAnnotator = variantAnnotator.getVariantAnnotatorProgram(); - List newSourceVersion = variantAnnotator.getVariantAnnotatorSourceVersion(); + ProjectMetadata.VariantAnnotationMetadata newAnnotationMetadata = variantAnnotator.getVariantAnnotationMetadata(); dbAdaptor.getMetadataManager().updateProjectMetadata(projectMetadata -> { - updateCurrentAnnotation(variantAnnotator, projectMetadata, overwrite, newAnnotator, newSourceVersion); + updateCurrentAnnotation(variantAnnotator, projectMetadata, overwrite, newAnnotationMetadata); return projectMetadata; }); } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManager.java index 523fb4c7a7b..c1681d402a4 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManager.java @@ -60,8 +60,8 @@ public abstract class VariantAnnotationManager { protected final VariantAnnotationMetadata checkCurrentAnnotation(VariantAnnotator annotator, ProjectMetadata projectMetadata, boolean overwrite) throws VariantAnnotatorException { - VariantAnnotatorProgram newAnnotator = annotator.getVariantAnnotatorProgram(); - List newSourceVersion = annotator.getVariantAnnotatorSourceVersion(); +// VariantAnnotatorProgram newAnnotator = annotator.getVariantAnnotatorProgram(); +// List newSourceVersion = annotator.getVariantAnnotatorSourceVersion(); // if (newSourceVersion == null) { // newSourceVersion = Collections.emptyList(); @@ -72,11 +72,12 @@ protected final VariantAnnotationMetadata checkCurrentAnnotation(VariantAnnotato // if (newSourceVersion.isEmpty()) { // throw new IllegalArgumentException("Missing annotator source version for VariantAnnotator: " + annotator.getClass()); // } - return checkCurrentAnnotation(projectMetadata, overwrite, newAnnotator, newSourceVersion); + ProjectMetadata.VariantAnnotationMetadata newVariantAnnotationMetadata = annotator.getVariantAnnotationMetadata(); + return checkCurrentAnnotation(projectMetadata, overwrite, newVariantAnnotationMetadata); } protected final VariantAnnotationMetadata checkCurrentAnnotation(ProjectMetadata projectMetadata, boolean overwrite, - VariantAnnotatorProgram newAnnotator, List newSourceVersion) + VariantAnnotationMetadata newVariantAnnotationMetadata) throws VariantAnnotatorException { VariantAnnotationMetadata current = projectMetadata.getAnnotation().getCurrent(); if (current == null) { @@ -88,6 +89,7 @@ protected final VariantAnnotationMetadata checkCurrentAnnotation(ProjectMetadata // Check using same annotator and same source version VariantAnnotatorProgram currentAnnotator = current.getAnnotator(); + VariantAnnotatorProgram newAnnotator = newVariantAnnotationMetadata.getAnnotator(); if (currentAnnotator != null && !currentAnnotator.equals(newAnnotator)) { String currentVersion = removePatchFromVersion(currentAnnotator.getVersion()); String newVersion = removePatchFromVersion(newAnnotator.getVersion()); @@ -113,24 +115,57 @@ protected final VariantAnnotationMetadata checkCurrentAnnotation(ProjectMetadata } } - List currentSourceVersion = current.getSourceVersion(); - if (CollectionUtils.isNotEmpty(currentSourceVersion) && !sameSourceVersion(newSourceVersion, currentSourceVersion)) { - String msg = "Source version of the annotator has changed. " - + "Existing annotation calculated with " - + currentSourceVersion.stream().map(ObjectMap::toJson).collect(Collectors.joining(" , ", "[ ", " ]")) - + ", attempting to annotate with " - + newSourceVersion.stream().map(ObjectMap::toJson).collect(Collectors.joining(" , ", "[ ", " ]")); + if (current.getDataRelease() != null && newVariantAnnotationMetadata.getDataRelease() == null) { + // Regression. DataRelease is lost. + String msg = "DataRelease missing. " + + "Existing annotation calculated with dataRelease " + current.getDataRelease().getRelease() + + ", attempting to annotate without explicit dataRelease"; if (overwrite) { logger.info(msg); } else { - // List of sources from cellbase 5.0.x is not reliable, and should - // not be taken into account to force a full annotation overwrite - if (newAnnotator.getName().toLowerCase().contains("cellbase") && newAnnotator.getVersion().startsWith("5.0")) { - logger.warn(msg); - logger.info("Ignore source version change at Cellbase v5.0.x"); + throw new VariantAnnotatorException(msg); + } + } + + if (newVariantAnnotationMetadata.getDataRelease() != null) { + if (current.getDataRelease() == null) { + // Missing current dataRelease. Continue. + } else { + if (!current.getDataRelease().equals(newVariantAnnotationMetadata.getDataRelease())) { + String msg = "DataRelease has changed. " + + "Existing annotation calculated with dataRelease " + current.getDataRelease().getRelease() + + ", attempting to annotate with " + newVariantAnnotationMetadata.getDataRelease().getRelease(); + + if (overwrite) { + logger.info(msg); + } else { + throw new VariantAnnotatorException(msg); + } + } + } + } else { + // Check sources for old cellbase versions + List currentSourceVersion = current.getSourceVersion(); + List newSourceVersion = newVariantAnnotationMetadata.getSourceVersion(); + if (CollectionUtils.isNotEmpty(currentSourceVersion) && !sameSourceVersion(newSourceVersion, currentSourceVersion)) { + String msg = "Source version of the annotator has changed. " + + "Existing annotation calculated with " + + currentSourceVersion.stream().map(ObjectMap::toJson).collect(Collectors.joining(" , ", "[ ", " ]")) + + ", attempting to annotate with " + + newSourceVersion.stream().map(ObjectMap::toJson).collect(Collectors.joining(" , ", "[ ", " ]")); + + if (overwrite) { + logger.info(msg); } else { - throw new VariantAnnotatorException(msg); + // List of sources from cellbase 5.0.x is not reliable, and should + // not be taken into account to force a full annotation overwrite + if (newAnnotator.getName().toLowerCase().contains("cellbase") && newAnnotator.getVersion().startsWith("5.0")) { + logger.warn(msg); + logger.info("Ignore source version change at Cellbase v5.0.x"); + } else { + throw new VariantAnnotatorException(msg); + } } } } @@ -156,15 +191,15 @@ private boolean sameSourceVersion(List newSourceVersion, List newSourceVersion = annotator.getVariantAnnotatorSourceVersion(); - updateCurrentAnnotation(annotator, projectMetadata, overwrite, newAnnotator, newSourceVersion); + updateCurrentAnnotation(annotator, projectMetadata, overwrite, annotator.getVariantAnnotationMetadata()); } + protected final void updateCurrentAnnotation(VariantAnnotator annotator, ProjectMetadata projectMetadata, - boolean overwrite, VariantAnnotatorProgram newAnnotator, - List newSourceVersion) + boolean overwrite, VariantAnnotationMetadata newAnnotationMetadata) throws VariantAnnotatorException { + List newSourceVersion = newAnnotationMetadata.getSourceVersion(); + VariantAnnotatorProgram newAnnotator = newAnnotationMetadata.getAnnotator(); if (newSourceVersion == null) { newSourceVersion = Collections.emptyList(); } @@ -174,10 +209,12 @@ protected final void updateCurrentAnnotation(VariantAnnotator annotator, Project if (newSourceVersion.isEmpty()) { throw new IllegalArgumentException("Missing annotator source version for VariantAnnotator: " + annotator.getClass()); } - checkCurrentAnnotation(projectMetadata, overwrite, newAnnotator, newSourceVersion); + checkCurrentAnnotation(projectMetadata, overwrite, newAnnotationMetadata); - projectMetadata.getAnnotation().getCurrent().setAnnotator(newAnnotator); - projectMetadata.getAnnotation().getCurrent().setSourceVersion(newSourceVersion); + VariantAnnotationMetadata current = projectMetadata.getAnnotation().getCurrent(); + current.setAnnotator(newAnnotator); + current.setSourceVersion(newSourceVersion); + current.setDataRelease(newAnnotationMetadata.getDataRelease()); } protected final VariantAnnotationMetadata registerNewAnnotationSnapshot(String name, VariantAnnotator annotator, @@ -204,7 +241,8 @@ protected final VariantAnnotationMetadata registerNewAnnotationSnapshot(String n name, Date.from(Instant.now()), current.getAnnotator(), - current.getSourceVersion()); + current.getSourceVersion(), + current.getDataRelease()); projectMetadata.getAnnotation().getSaved().add(newSnapshot); // Increment ID of the current annotation diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/AbstractCellBaseVariantAnnotator.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/AbstractCellBaseVariantAnnotator.java index 91d127418c7..cd06618de38 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/AbstractCellBaseVariantAnnotator.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/AbstractCellBaseVariantAnnotator.java @@ -55,6 +55,7 @@ public abstract class AbstractCellBaseVariantAnnotator extends VariantAnnotator protected final String species; protected final String assembly; protected final String cellbaseVersion; + protected final String cellbaseDataRelease; protected final QueryOptions queryOptions; protected final boolean supportImpreciseVariants; protected final boolean supportStarAlternate; @@ -69,6 +70,7 @@ public AbstractCellBaseVariantAnnotator(StorageConfiguration storageConfiguratio species = projectMetadata.getSpecies(); assembly = projectMetadata.getAssembly(); cellbaseVersion = storageConfiguration.getCellbase().getVersion(); + cellbaseDataRelease = storageConfiguration.getCellbase().getDataRelease(); queryOptions = new QueryOptions(); if (StringUtils.isNotEmpty(params.getString(VariantStorageOptions.ANNOTATOR_CELLBASE_INCLUDE.key()))) { @@ -98,6 +100,7 @@ public AbstractCellBaseVariantAnnotator(StorageConfiguration storageConfiguratio + ':' + (variant.getAlternate().isEmpty() ? "-" : variant.getAlternate()); } checkNotNull(cellbaseVersion, "cellbase version"); +// checkNotNull(cellbaseDataRelease, "cellbase dataRelease"); checkNotNull(species, "species"); checkNotNull(assembly, "assembly"); @@ -109,15 +112,6 @@ protected static void checkNotNull(String value, String name) throws VariantAnno } } - public static String toCellBaseSpeciesName(String scientificName) { - if (scientificName != null && scientificName.contains(" ")) { - String[] split = scientificName.split(" ", 2); - scientificName = (split[0].charAt(0) + split[1]).toLowerCase(); - } - return scientificName; - } - - @Override public final List annotate(List variants) throws VariantAnnotatorException { List filteredVariants = filterVariants(variants); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/CellBaseRestVariantAnnotator.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/CellBaseRestVariantAnnotator.java index 3a78b6f4e8f..c4b754ec87f 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/CellBaseRestVariantAnnotator.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/CellBaseRestVariantAnnotator.java @@ -21,6 +21,7 @@ import org.opencb.biodata.models.variant.avro.VariantAnnotation; import org.opencb.cellbase.client.config.ClientConfiguration; import org.opencb.cellbase.client.rest.CellBaseClient; +import org.opencb.cellbase.core.models.DataRelease; import org.opencb.cellbase.core.result.CellBaseDataResponse; import org.opencb.cellbase.core.result.CellBaseDataResult; import org.opencb.commons.datastore.core.Event; @@ -28,6 +29,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; +import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.annotation.VariantAnnotatorException; @@ -46,6 +48,7 @@ public class CellBaseRestVariantAnnotator extends AbstractCellBaseVariantAnnotator { private final CellBaseClient cellBaseClient; + private final CellBaseUtils cellBaseUtils; public CellBaseRestVariantAnnotator(StorageConfiguration storageConfiguration, ProjectMetadata projectMetadata, ObjectMap options) throws VariantAnnotatorException { @@ -64,10 +67,15 @@ public CellBaseRestVariantAnnotator(StorageConfiguration storageConfiguration, P VariantStorageOptions.ANNOTATION_TIMEOUT.defaultValue()); clientConfiguration.getRest().setTimeout(timeoutMillis); - cellBaseClient = new CellBaseClient(species, assembly, clientConfiguration); + cellBaseClient = new CellBaseClient(species, assembly, cellbaseDataRelease, clientConfiguration); + cellBaseUtils = new CellBaseUtils(cellBaseClient); + logger.info("Annotating with Cellbase REST. {}", cellBaseUtils); - logger.info("Annotating with Cellbase REST. host '{}', version '{}', species '{}', assembly '{}'", - cellbaseRest, cellbaseVersion, species, assembly); + try { + cellBaseUtils.validateCellBaseConnection(); + } catch (IOException e) { + throw new VariantAnnotatorException(e.getMessage(), e); + } } @@ -88,7 +96,29 @@ protected List> annotateFiltered(List String.valueOf(dr.getRelease()).equals(cellBaseClient.getDataRelease())) + .findFirst() + .orElse(null); + } + } catch (IOException e) { + throw new VariantAnnotatorException("Error fetching CellBase information from " + + getDebugInfo("/meta/" + species + "/dataReleases") + ". "); + } + return new ProjectMetadata.VariantAnnotationMetadata(-1, null, null, + getVariantAnnotatorProgram(), + getVariantAnnotatorSourceVersion(), + dataRelease); + } + + private ProjectMetadata.VariantAnnotatorProgram getVariantAnnotatorProgram() throws VariantAnnotatorException { CellBaseDataResponse response; try { response = cellBaseClient.getMetaClient().about(); @@ -121,9 +151,7 @@ public ProjectMetadata.VariantAnnotatorProgram getVariantAnnotatorProgram() thro return program; } - - @Override - public List getVariantAnnotatorSourceVersion() throws VariantAnnotatorException { + private List getVariantAnnotatorSourceVersion() throws VariantAnnotatorException { CellBaseDataResponse response; try { response = cellBaseClient.getMetaClient().versions(); @@ -151,11 +179,8 @@ public List getVariantAnnotatorSourceVersion() throws VariantAnnotato } private String getDebugInfo(String path) { - return "host: '" + cellBaseClient.getClientConfiguration().getRest().getHosts().get(0) + "', " - + "version: '" + cellBaseClient.getClientConfiguration().getVersion() + "', " - + "species: '" + species + "', " - + "assembly: '" + assembly + "', " - + "path: '" + path + "'"; + return cellBaseUtils.toString() + + " path: '" + path + "'"; } private Event errorEvent(CellBaseDataResponse response) { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotator.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotator.java index 32d96ee6377..084a834fc13 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotator.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotator.java @@ -43,8 +43,6 @@ public VariantAnnotator(StorageConfiguration configuration, ProjectMetadata proj */ public abstract List annotate(List variants) throws VariantAnnotatorException; - public abstract ProjectMetadata.VariantAnnotatorProgram getVariantAnnotatorProgram() throws VariantAnnotatorException; - - public abstract List getVariantAnnotatorSourceVersion() throws VariantAnnotatorException; + public abstract ProjectMetadata.VariantAnnotationMetadata getVariantAnnotationMetadata() throws VariantAnnotatorException; } diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorFactory.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorFactory.java index 935b91d2832..835e6c561ce 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorFactory.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VariantAnnotatorFactory.java @@ -39,7 +39,7 @@ public final class VariantAnnotatorFactory { public enum AnnotationEngine { CELLBASE_REST, CELLBASE, - VEP, +// VEP, OTHER } @@ -73,8 +73,8 @@ public static VariantAnnotator buildVariantAnnotator(StorageConfiguration config case CELLBASE_REST: case CELLBASE: return new CellBaseRestVariantAnnotator(configuration, projectMetadata, options); - case VEP: - return VepVariantAnnotator.buildVepAnnotator(); +// case VEP: +// return VepVariantAnnotator.buildVepAnnotator(); case OTHER: String className = options.getString(VariantStorageOptions.ANNOTATOR_CLASS.key()); logger.info("Annotating with {} = {}", annotationEngine, className); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VepVariantAnnotator.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VepVariantAnnotator.java index 44bca0255e9..250194f0003 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VepVariantAnnotator.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/annotation/annotators/VepVariantAnnotator.java @@ -21,9 +21,7 @@ import org.opencb.biodata.formats.variant.annotation.io.VepFormatReader; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.VariantAnnotation; -import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.annotation.VariantAnnotatorException; @@ -41,7 +39,7 @@ /** * Created by fjlopez on 10/04/15. */ -public class VepVariantAnnotator extends VariantAnnotator { +public abstract class VepVariantAnnotator extends VariantAnnotator { private final JsonFactory factory; private ObjectMapper jsonObjectMapper; @@ -54,13 +52,13 @@ public VepVariantAnnotator() throws VariantAnnotatorException { jsonObjectMapper.addMixIn(VariantAnnotation.class, VariantAnnotationMixin.class); } - public static VepVariantAnnotator buildVepAnnotator() { - try { - return new VepVariantAnnotator(); - } catch (VariantAnnotatorException ignore) { - return null; - } - } +// public static VepVariantAnnotator buildVepAnnotator() { +// try { +// return new VepVariantAnnotator(); +// } catch (VariantAnnotatorException ignore) { +// return null; +// } +// } private static void checkNull(String value, String name) throws VariantAnnotatorException { if (value == null || value.isEmpty()) { @@ -75,16 +73,6 @@ public List annotate(List variants) throws VariantAn return null; } - @Override - public ProjectMetadata.VariantAnnotatorProgram getVariantAnnotatorProgram() { - return null; - } - - @Override - public List getVariantAnnotatorSourceVersion() { - return null; - } - /////// LOAD ANNOTATION public void loadAnnotation(final VariantDBAdaptor variantDBAdaptor, final URI uri, QueryOptions options) throws IOException { diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java index 7231bcbaff1..74ee51bbe3a 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/search/solr/VariantSearchManager.java @@ -45,6 +45,7 @@ import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.storage.core.exceptions.VariantSearchException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; +import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.iterators.VariantDBIterator; @@ -83,10 +84,10 @@ public class VariantSearchManager { public static final int DEFAULT_INSERT_BATCH_SIZE = 10000; public VariantSearchManager(VariantStorageMetadataManager variantStorageMetadataManager, - StorageConfiguration storageConfiguration, ObjectMap options) { + CellBaseUtils cellBaseUtils, StorageConfiguration storageConfiguration, ObjectMap options) { this.solrQueryParser = new SolrQueryParser(variantStorageMetadataManager); - this.cellBaseClient = new CellBaseClient(storageConfiguration.getCellbase().toClientConfiguration()); + this.cellBaseClient = cellBaseUtils.getCellBaseClient(); this.options = options; this.variantSearchToVariantConverter = new VariantSearchToVariantConverter(); this.configSet = storageConfiguration.getSearch().getConfigSet(); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java index d2a7fe380e6..f936e9f894e 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/utils/CellBaseUtilsTest.java @@ -7,6 +7,7 @@ import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; import org.opencb.biodata.models.core.Region; +import org.opencb.biodata.models.core.Transcript; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.ConsequenceType; import org.opencb.biodata.models.variant.avro.VariantAnnotation; @@ -25,7 +26,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import static org.junit.Assert.*; @@ -48,13 +48,15 @@ public class CellBaseUtilsTest { @Parameters(name = "{0}") public static List data() { return Arrays.asList( - new Object[]{"http://ws.opencb.org/cellbase-4.7.3/", "v4", "grch37"}, - new Object[]{"http://ws.opencb.org/cellbase-4.8.2/", "v4", "grch37"}, -// new Object[]{"http://ws.opencb.org/cellbase-4.8.3/", "v4", "grch37"}, -// new Object[]{"http://ws.opencb.org/cellbase-4.9.0/", "v4", "grch37"}, -// new Object[]{"http://ws.opencb.org/cellbase/", "v4", "grch37"}, - new Object[]{"https://ws.zettagenomics.com/cellbase/", "v5", "grch38"}, - new Object[]{"https://ws.zettagenomics.com/cellbase/", "v5.1", "grch38"}); + new Object[]{"http://ws.opencb.org/cellbase-4.7.3/", "v4", "grch37", null}, + new Object[]{"http://ws.opencb.org/cellbase-4.8.2/", "v4", "grch37", null}, +// new Object[]{"http://ws.opencb.org/cellbase-4.8.3/", "v4", "grch37", null}, +// new Object[]{"http://ws.opencb.org/cellbase-4.9.0/", "v4", "grch37", null}, +// new Object[]{"http://ws.opencb.org/cellbase/", "v4", "grch37", null}, + new Object[]{"https://uk.ws.zettagenomics.com/cellbase/", "v5.2", "grch37", "1"}, + new Object[]{"https://ws.zettagenomics.com/cellbase/", "v5", "grch38", null}, + new Object[]{"https://ws.zettagenomics.com/cellbase/", "v5.1", "grch38", "1"}, + new Object[]{"https://ws.zettagenomics.com/cellbase/", "v5.1", "grch38", "2"}); } @Parameter(0) @@ -66,12 +68,20 @@ public static List data() { @Parameter(2) public String assembly; + @Parameter(3) + public String dataRelease; + @Before public void setUp() throws Exception { - cellBaseClient = new CellBaseClient("hsapiens", assembly, + cellBaseClient = new CellBaseClient("hsapiens", assembly, dataRelease, new ClientConfiguration().setVersion(version) .setRest(new RestConfig(Collections.singletonList(url), 10000))); - cellBaseUtils = new CellBaseUtils(cellBaseClient, assembly); + cellBaseUtils = new CellBaseUtils(cellBaseClient); + } + + @Test + public void testValidateCellBaseConnection() throws IOException { + cellBaseUtils.validateCellBaseConnection(); } @Test @@ -104,14 +114,21 @@ public void testGetGeneByTranscriptId() { } @Test - public void testGetGeneByTranscriptName() { + public void testGetGeneByTranscriptName() throws IOException { Assume.assumeTrue(version.startsWith("v5")); - assertNotNull(cellBaseUtils.getGeneRegion(Arrays.asList("BRCA2-206"), false).get(0)); + String transcriptName = cellBaseUtils.getCellBaseClient().getGeneClient().get(Collections.singletonList("BRCA2"), new QueryOptions()).firstResult().getTranscripts() + .stream() + .map(Transcript::getName) + .filter(t -> t.startsWith("BRCA2-")) + .findFirst() + .orElse(null); + assertNotNull(cellBaseUtils.getGeneRegion(Arrays.asList(transcriptName), false).get(0)); } @Test public void convertGeneToRegionHGNC() { Assume.assumeTrue(version.startsWith("v5")); + Assume.assumeFalse("HGNC ids not supported in GRCH37", assembly.equalsIgnoreCase("grch37")); assertNotNull(cellBaseUtils.getGeneRegion("HGNC:12363")); } @@ -152,6 +169,7 @@ public void convertGeneToRegionFail() { @Test public void validateGeneNames() { Assume.assumeTrue(!version.startsWith("v4")); + Assume.assumeFalse("HGNC ids not supported in GRCH37", assembly.equalsIgnoreCase("grch37")); List validated = cellBaseUtils.validateGenes(Arrays.asList("BRCA2", "NonExistingGene", "HGNC:12363"), true); assertEquals(Arrays.asList("BRCA2", "TSC2"), validated); } @@ -184,6 +202,7 @@ public void testGetMeta() throws IOException { @Test public void testGetTranscriptFlags() throws IOException { + Assume.assumeTrue("The variant tested here is from GRCH38", assembly.equalsIgnoreCase("grch38")); CellBaseDataResponse v = cellBaseClient.getVariantClient() .getAnnotationByVariantIds(Collections.singletonList("1:26644214:T:C"), new QueryOptions()); VariantAnnotation variantAnnotation = v.firstResult(); diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java index 58cbac878f3..71f5135e08c 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/DummyTestAnnotator.java @@ -9,7 +9,6 @@ import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; import org.opencb.opencga.storage.core.variant.annotation.annotators.VariantAnnotator; -import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -57,12 +56,10 @@ public List annotate(List variants) throws VariantAn } @Override - public ProjectMetadata.VariantAnnotatorProgram getVariantAnnotatorProgram() { - return new ProjectMetadata.VariantAnnotatorProgram("MyAnnotator", key, null); + public ProjectMetadata.VariantAnnotationMetadata getVariantAnnotationMetadata() throws VariantAnnotatorException { + return new ProjectMetadata.VariantAnnotationMetadata(-1, null, null, + new ProjectMetadata.VariantAnnotatorProgram("MyAnnotator", key, null), + Collections.singletonList(new ObjectMap("data", "genes")), null); } - @Override - public List getVariantAnnotatorSourceVersion() { - return Collections.singletonList(new ObjectMap("data", "genes")); - } } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManagerTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManagerTest.java index 3796aa13803..4ff3a309030 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManagerTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/annotation/VariantAnnotationManagerTest.java @@ -5,6 +5,7 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; @@ -80,6 +81,72 @@ public void testChangeAnnotatorFail() throws Exception { assertEquals("v2", variantStorageEngine.getMetadataManager().getProjectMetadata().getAnnotation().getCurrent().getAnnotator().getVersion()); } + @Test + public void testChangeDataRelease() throws Exception { + VariantStorageEngine variantStorageEngine = getVariantStorageEngine(); + variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); + variantStorageEngine.getConfiguration().getCellbase().setVersion("v5"); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(null); + variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); + variantStorageEngine.reloadCellbaseConfiguration(); + variantStorageEngine.getCellBaseUtils().validateCellBaseConnection(); + + runDefaultETL(smallInputUri, variantStorageEngine, newStudyMetadata(), + new ObjectMap(VariantStorageOptions.ANNOTATE.key(), false)); + + // First annotation. Should run ok. + variantStorageEngine.annotate(outputUri, new ObjectMap()); + assertNull(variantStorageEngine.getMetadataManager().getProjectMetadata().getAnnotation().getCurrent().getDataRelease()); + + variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.1"); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease("1"); + variantStorageEngine.reloadCellbaseConfiguration(); + + // New annotator. Do not overwrite. Should fail. + try { + variantStorageEngine.annotate(outputUri, new ObjectMap(VariantStorageOptions.ANNOTATION_OVERWEITE.key(), false)); + fail("Should fail"); + } catch (Exception e) { + e.printStackTrace(); + } + + // New annotator. Overwrite. Should run ok. + variantStorageEngine.annotate(outputUri, new ObjectMap(VariantStorageOptions.ANNOTATION_OVERWEITE.key(), true)); + assertEquals("1", String.valueOf(variantStorageEngine.getMetadataManager().getProjectMetadata().getAnnotation().getCurrent().getDataRelease().getRelease())); + + + variantStorageEngine.getConfiguration().getCellbase().setDataRelease("2"); + variantStorageEngine.reloadCellbaseConfiguration(); + + // Same annotator, new datarelease. Do not overwrite. Should fail. + try { + variantStorageEngine.annotate(outputUri, new ObjectMap(VariantStorageOptions.ANNOTATION_OVERWEITE.key(), false)); + fail("Should fail"); + } catch (Exception e) { + e.printStackTrace(); + assertEquals("DataRelease has changed. Existing annotation calculated with dataRelease 1, attempting to annotate with 2", e.getMessage()); + } + + // Same annotator, new datarelease. Overwrite. Should run ok. + variantStorageEngine.annotate(outputUri, new ObjectMap(VariantStorageOptions.ANNOTATION_OVERWEITE.key(), true)); + assertEquals("2", String.valueOf(variantStorageEngine.getMetadataManager().getProjectMetadata().getAnnotation().getCurrent().getDataRelease().getRelease())); + + // Revert annotator to 5.0. Do not overwrite. Should fail. + variantStorageEngine.getConfiguration().getCellbase().setDataRelease(null); + variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.0"); + variantStorageEngine.reloadCellbaseConfiguration(); + try { + variantStorageEngine.annotate(outputUri, new ObjectMap(VariantStorageOptions.ANNOTATION_OVERWEITE.key(), false)); + fail("Should fail"); + } catch (Exception e) { + e.printStackTrace(); + } + + // Revert annotator to 5.0. Do not overwrite. Should run ok. + variantStorageEngine.annotate(outputUri, new ObjectMap(VariantStorageOptions.ANNOTATION_OVERWEITE.key(), true)); + assertNull(variantStorageEngine.getMetadataManager().getProjectMetadata().getAnnotation().getCurrent().getDataRelease()); + } + @Test public void testMultiAnnotations() throws Exception { diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParserTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParserTest.java index d9a7065c4ec..ea3ee11c1b9 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParserTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/query/VariantQueryParserTest.java @@ -44,7 +44,7 @@ public class VariantQueryParserTest implements DummyVariantStorageTest { public void setUp() throws Exception { metadataManager = new VariantStorageMetadataManager(new DummyVariantStorageMetadataDBAdaptorFactory(true)); ClientConfiguration clientConfiguration = getVariantStorageEngine().getConfiguration().getCellbase().toClientConfiguration(); - CellBaseUtils cellBaseUtils = new CellBaseUtils(new CellBaseClient("hsapiens", "grch38", clientConfiguration), "grch38"); + CellBaseUtils cellBaseUtils = new CellBaseUtils(new CellBaseClient("hsapiens", "grch38", clientConfiguration)); variantQueryParser = new VariantQueryParser(cellBaseUtils, metadataManager); diff --git a/opencga-storage/opencga-storage-core/src/test/resources/2019_01_10_all_PCAWG_sigs_rearr.tsv b/opencga-storage/opencga-storage-core/src/test/resources/2019_01_10_all_PCAWG_sigs_rearr.tsv new file mode 100644 index 00000000000..ad3a81101b4 --- /dev/null +++ b/opencga-storage/opencga-storage-core/src/test/resources/2019_01_10_all_PCAWG_sigs_rearr.tsv @@ -0,0 +1,33 @@ +Biliary_A (RefSig R4) Biliary_B (RefSig R10) Biliary_C (RefSig R2+R5) Biliary_D (RefSig R8) Biliary_E (RefSig R6b) Biliary_F (RefSig R6a) Bladder_A (RefSig R17) Bladder_B (RefSig R2+R11) Bladder_C (RefSig R10) Bladder_D (RefSig R7) Bladder_E (RefSig R4+R6a) Bone_SoftTissue_A (RefSig R11) Bone_SoftTissue_B (RefSig R4) Bone_SoftTissue_C (RefSig R8) Bone_SoftTissue_D (RefSig R7) Bone_SoftTissue_E (RefSig R6b) Bone_SoftTissue_F (RefSig R6a) Bone_SoftTissue_G (RefSig R2) Bone_SoftTissue_H (RefSig R1) Breast_A (RefSig R1) Breast_B (RefSig R6b) Breast_C (RefSig R8) Breast_D (RefSig R5) Breast_E (RefSig R6a) Breast_F (RefSig R2+R11) Breast_G (RefSig R3) Breast_H (RefSig R4) Cervix_A (RefSig R5+R8) Cervix_B (RefSig R10) CNS_A (RefSig R4+R6b) CNS_B (RefSig R2+R1) CNS_C (RefSig R6a) CNS_D (RefSig R20) Colorectal_A (RefSig R2) Colorectal_B (RefSig R3) Colorectal_C (RefSig R7) Colorectal_D (RefSig R6a+R6b) Esophagus_A (RefSig R1) Esophagus_B (RefSig R12) Esophagus_C (RefSig R2+R9) Esophagus_D (RefSig R4) Esophagus_E (RefSig R7) Esophagus_F (RefSig R6a) Esophagus_G (RefSig R2+R11) Head_neck_A (RefSig R16) Head_neck_B (RefSig R5+R3) Head_neck_C (RefSig R2+R8) Kidney_A (RefSig R7) Kidney_B (RefSig R6b) Kidney_C (RefSig R4) Kidney_D (RefSig R2) Kidney_E (RefSig R6a) Liver_A (RefSig R5) Liver_B (RefSig R4) Liver_C (RefSig R1) Liver_D (RefSig R2) Liver_E (RefSig R3) Liver_F (RefSig R8) Liver_G (RefSig R6a+R6b) Lung_A (RefSig R1) Lung_B (RefSig R2) Lung_C (RefSig R9) Lung_D (RefSig R3) Lung_E (RefSig R13) Lung_F (RefSig R11) Lung_G (RefSig R6a) Lung_H (RefSig R6b) Lung_I (RefSig R4) Lung_J (RefSig R7) Lymphoid_A (RefSig R2) Lymphoid_B (RefSig R4+R6a) Lymphoid_C (RefSig R1) Lymphoid_D (RefSig R6a+R6b) Lymphoid_E (RefSig R5) Ovary_A (RefSig R5) Ovary_B (RefSig R6a+R6b) Ovary_C (RefSig R1) Ovary_D (RefSig R18) Ovary_E (RefSig R14) Ovary_F (RefSig R2+R7) Ovary_G (RefSig R3) Pancreas_A (RefSig R1) Pancreas_B (RefSig R7) Pancreas_C (RefSig R9) Pancreas_D (RefSig R2) Pancreas_E (RefSig R4) Pancreas_F (RefSig R12) Pancreas_G (RefSig R6a) Prostate_A (RefSig R6b) Prostate_B (RefSig R2) Prostate_C (RefSig R8) Prostate_D (RefSig R1) Prostate_E (RefSig R6a) Prostate_F (RefSig R3) Prostate_G (RefSig R9) Prostate_H (RefSig R4) Prostate_I (RefSig R15) Skin_A (RefSig R6b) Skin_B (RefSig R11) Skin_C (RefSig R1) Skin_D (RefSig R6a) Skin_E (RefSig R4) Skin_F (RefSig R2) Skin_G (RefSig R5) Stomach_A (RefSig R6a) Stomach_B (RefSig R2) Stomach_C (RefSig R1) Stomach_D (RefSig R1+R3) Stomach_E (RefSig R7) Stomach_F (RefSig R6b) Stomach_G (RefSig R4) Stomach_H (RefSig R9) Uterus_A (RefSig R19) Uterus_B (RefSig R5+R3) Uterus_C (RefSig R8) Uterus_D (RefSig R2+R1) +clustered_del_1-10Kb 0.0219240395346361 8.96489862652724e-20 0.0129411462384023 1.22619698724958e-19 1.19882047685856e-05 1.10123020060586e-19 0.00163447966962098 1.48334001522991e-19 1.77069036322592e-19 0.000993300748788492 0.0265961913442519 1.73644761214785e-20 0.00840235849787134 3.47114419294204e-20 2.53995730561859e-20 0.00257292682767552 0.00421478088921572 2.09258714259033e-20 0.00164231346261066 2.23326137118201e-20 0.0185481275216943 0.00173280497159421 0.00586566812286373 2.20467501238808e-20 1.55467333841473e-20 1.4436164942841e-20 0.00266668636595723 0.0332876506344296 0.00470665759099834 0.00880017814608158 0.000321776472396562 7.83843969998703e-09 0.0193961895385854 0.00535168792803461 0.00104877455120512 0.0122691976757478 0.00210612383999814 2.23979131450565e-08 0.025195036385617 0.00532328015804634 9.38762695860971e-20 0.00859311447286751 0.0151454053396775 0.00559201783063035 0.0154630458359147 1.05461989671689e-19 1.2289108304478e-19 0.00367077737158173 0.0166155895009217 1.58929435340401e-20 0.00982028558170012 2.20179740219242e-20 3.19927602094556e-20 0.0154113475975854 4.1974369108772e-20 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 0.00297177121636871 5.59117980134354e-20 3.07572841045254e-20 0.0122216001001295 3.95236128409755e-20 0.0364854596236851 3.66066275854242e-20 3.8645359827963e-20 3.9024530190521e-06 0.00144765685013369 3.66979116100265e-20 2.71318234348475e-20 0.0112786270391133 0.00450924836566096 3.9862459500132e-20 0.00142387266050199 0.00839669436385418 0.0087382775861428 0.000615784930604334 0.0193330713892071 6.96663918906589e-20 5.74590300026141e-20 4.25834079286928e-20 9.07006048511931e-20 9.26847697415676e-20 0.00534774643091108 1.29829956957131e-09 0.00576887461066845 0.0275496479895544 8.45408369612758e-20 0.0310517235915492 4.19620688953765e-20 7.39595841870445e-20 0.00377374798056132 0.00936750897466945 9.14098254498316e-20 5.82597787560824e-20 0.00331738945958438 2.328900718833e-16 0.0131514774020814 3.61128385116762e-20 0.00530882113299865 0.00546197380750563 3.33025645875325e-20 2.90817268902941e-20 3.57892404846891e-20 0.0133303268440946 0.0024216224845211 5.04326126337211e-05 3.74763307217287e-20 0.00772376647689791 0.02086936664392 0.00118316047987935 2.94386643348713e-20 0.00888646883633611 3.87919548021642e-20 4.24615471433568e-20 2.3589125068214e-20 +clustered_del_10-100Kb 0.0029720186877007 8.96489862652724e-20 1.02780470407957e-19 1.22619698724958e-19 0.0163699456520021 1.10123020060586e-19 1.8733134051656e-19 0.00685396014797732 0.011292541493584 0.0116954303672147 1.84029395221006e-19 1.73644761214785e-20 0.00636066918541737 3.47114419294204e-20 0.0205732300113116 0.0163694206939208 0.0126070062796838 2.09258714259033e-20 2.55596049694926e-20 0.00125586429840823 0.0195652151497073 2.82643086695692e-20 0.00131530165381406 2.20467501238808e-20 1.14388749423539e-09 1.4436164942841e-20 0.007848119365188 0.0199725903806578 2.80063500983306e-19 0.00130007672338614 1.47092264292995e-19 0.00331943267798072 0.0499733727186174 5.38720357252812e-20 0.0136199796947498 0.0339464257798362 5.54797685328391e-20 0.00920521760393713 8.23014850574395e-20 0.00698938490376017 0.0047038730295443 0.0275916346949569 1.92801365724734e-17 0.000196855919336178 0.00681082810305503 1.05461989671689e-19 0.0150276264271617 2.25756462441701e-20 0.0496122913779074 0.00102102579445372 2.2439059470699e-20 2.20179740219242e-20 3.19927602094556e-20 0.0299225970068985 4.1974369108772e-20 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 0.00749905620296839 5.59117980134354e-20 3.07572841045254e-20 3.1201016757648e-20 0.00232987540033247 0.0163769577392175 3.66066275854242e-20 0.00676063371612298 4.72458028745174e-20 0.00139528990126099 0.0140663501813155 2.71318234348475e-20 0.00160837719051981 0.00379027212707149 0.00850629476796414 2.07537436576165e-06 0.000839743273742038 0.0169500275147236 0.004979418716889 0.0201264073370955 0.00269004978871228 5.74590300026141e-20 4.25834079286928e-20 0.00143160341821524 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 0.0280579660469988 8.45408369612758e-20 0.0448612506999436 4.19620688953765e-20 7.39595841870445e-20 0.00137062285329386 0.00743487266519204 9.14098254498316e-20 5.82597787560824e-20 0.00530854509418822 0.00357021192643765 0.0101674818058375 3.61128385116762e-20 4.81650909640983e-20 0.00477622303855236 0.00351695843799513 2.90817268902941e-20 0.0136255738278364 0.0224452528749721 3.1895005854227e-20 0.00743208823752568 3.74763307217287e-20 0.0281099892098995 0.00953794329534733 4.40547697474683e-20 2.94386643348713e-20 0.0143178637992704 0.00707247932184399 0.00259055587461974 2.3589125068214e-20 +clustered_del_100Kb-1Mb 0.00912870467747687 0.00611491387780227 0.00660043700572228 1.22619698724958e-19 0.0252807487030364 1.10123020060586e-19 0.029002843801801 0.00756977041439724 1.77069036322592e-19 0.0369913808344598 0.00319984879254965 1.73644761214785e-20 0.00334291818942687 3.47114419294204e-20 0.037028013419234 0.0565075374288703 0.00873836675188611 2.09258714259033e-20 0.025213915771966 7.76843937774752e-18 0.0583177662352634 0.00271108864540199 0.00089225450181873 0.00422374907057023 1.55467333841473e-20 1.4436164942841e-20 0.00796574643693194 2.26501583960149e-19 0.010185110248146 0.00448158972597429 1.47092264292995e-19 1.71132737562608e-19 0.126992554724737 5.38720357252812e-20 0.016136633717675 0.0900447872801972 0.0137991630653258 7.08490153865265e-20 0.0388780922975985 7.55364316665385e-20 9.38762695860971e-20 0.0396356071836012 0.0136219721858698 0.00723033859605504 0.0256336178497246 0.00317194546614679 1.2289108304478e-19 0.0325508136569761 0.0126024359445652 0.0133917000701513 2.2439059470699e-20 0.00993721825323664 3.19927602094556e-20 0.0373141849353869 0.00062416625146214 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 0.019319622691313 5.59117980134354e-20 0.00527783916075073 3.1201016757648e-20 3.95236128409755e-20 0.0651291232964312 3.66066275854242e-20 0.0164928982016403 0.0363250741061232 0.0136863876191219 0.0013871628348491 2.71318234348475e-20 0.00525704975780397 2.19355013036344e-20 0.0659803773822153 2.42417308498371e-20 4.71892355961059e-20 0.0243636614990173 0.0039628925858577 0.0293699410567715 0.000869118404523076 5.74590300026141e-20 4.25834079286928e-20 1.06195458538606e-10 0.00226947707249235 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 0.0783713439655967 0.0146788007695499 0.0704268330093657 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 0.0118136301118692 9.14098254498316e-20 5.82597787560824e-20 0.0104948831587225 3.85987478930723e-20 0.0567464530882794 3.61128385116762e-20 5.23518594848722e-06 0.00664756469377417 0.00908475265869715 2.90817268902941e-20 3.57892404846891e-20 0.0291665507765977 0.0111671978898813 0.00922243695811348 3.74763307217287e-20 0.0406883069744519 0.048931509529538 0.00757034241439712 2.94386643348713e-20 0.0237361667483182 3.87919548021642e-20 4.24615471433568e-20 2.3589125068214e-20 +clustered_del_1Mb-10Mb 0.024825217399492 8.96489862652724e-20 1.02780470407957e-19 5.25795056441111e-12 0.088318717899866 0.011998448220339 1.8733134051656e-19 0.00550309699238198 1.77069036322592e-19 2.18598846219008e-19 0.0573019717213734 0.0296982878867971 2.98309639014467e-08 3.47114419294204e-20 2.53995730561859e-20 0.184505170751191 0.00310159165455344 2.09258714259033e-20 2.55596049694926e-20 2.23326137118201e-20 0.132783783498158 2.82643086695692e-20 1.77712011350644e-20 0.0178840111828111 1.55467333841473e-20 1.4436164942841e-20 6.11036934902121e-20 2.26501583960149e-19 0.0219477898988197 0.127364342547627 1.47092264292995e-19 0.0454310874128393 0.0240831354610727 5.38720357252812e-20 5.90949151544599e-20 4.91432899251023e-20 0.0626673207856063 7.08490153865265e-20 0.0624634277379155 7.55364316665385e-20 9.38762695860971e-20 7.36136279388596e-20 0.0475955651956634 1.338426986118e-05 0.0497506692112039 1.05461989671689e-19 1.2289108304478e-19 2.25756462441701e-20 0.125316039633714 0.0135606030618116 2.2439059470699e-20 0.0784009994099736 3.19927602094556e-20 0.0246862610859659 4.1974369108772e-20 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 0.0786198396932048 5.59117980134354e-20 3.07572841045254e-20 3.1201016757648e-20 3.95236128409755e-20 0.0207021630399003 3.66066275854242e-20 0.0296724929839326 0.114585146672656 0.0176534748837542 3.66979116100265e-20 2.71318234348475e-20 0.0257187941625285 2.19355013036344e-20 0.0491912480088188 2.42417308498371e-20 0.00103692512680485 0.0767182448802261 5.64473945464854e-20 0.00891766295343186 0.00233202332562281 5.74590300026141e-20 4.25834079286928e-20 9.07006048511931e-20 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 0.0135325347037109 0.0698223112148756 0.0525217363521345 0.118585879686628 4.19620688953765e-20 7.39595841870445e-20 0.00596548202316543 0.0181015171463174 9.14098254498316e-20 5.82597787560824e-20 4.80460431933668e-20 3.85987478930723e-20 0.158577816607509 3.61128385116762e-20 4.81650909640983e-20 0.0292621668556268 0.00417752099510993 2.90817268902941e-20 3.57892404846891e-20 0.0449642721835367 3.1895005854227e-20 2.8220400790683e-20 3.74763307217287e-20 3.10815414898578e-20 0.0915970116325852 5.50659833550121e-07 2.94386643348713e-20 0.0100627058394856 0.00251364011612926 4.24615471433568e-20 2.3589125068214e-20 +clustered_del_>10Mb 0.0335595477151976 8.96489862652724e-20 1.02780470407957e-19 1.38739146578111e-16 0.01436342612359 0.14947014869385 0.030488442283497 0.00965637592425178 1.77069036322592e-19 0.00620364875878565 0.0196421243263923 0.00978191854770979 8.14578518630083e-05 3.47114419294204e-20 2.53995730561859e-20 1.67991834076044e-20 0.212602525720689 2.09258714259033e-20 2.55596049694926e-20 2.23326137118201e-20 1.87723865406083e-20 2.82643086695692e-20 1.77712011350644e-20 0.197376484120235 1.55467333841473e-20 1.4436164942841e-20 1.90694174609194e-20 2.26501583960149e-19 0.00877911595952787 1.56314289955482e-19 1.47092264292995e-19 0.14648066317054 1.54196727801832e-19 5.38720357252812e-20 5.90949151544599e-20 4.91432899251023e-20 0.104051400549686 2.09515325439118e-15 8.23014850574395e-20 7.55364316665385e-20 9.38762695860971e-20 7.36136279388596e-20 0.110575380932118 7.50149635663954e-20 0.0302537853311375 1.05461989671689e-19 1.2289108304478e-19 2.25756462441701e-20 2.86083723118323e-20 1.58929435340401e-20 2.2439059470699e-20 0.165794022782313 3.19927602094556e-20 0.00341287498534154 4.1974369108772e-20 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 0.129127852072371 5.59117980134354e-20 3.07572841045254e-20 3.1201016757648e-20 3.95236128409755e-20 1.29392135228993e-05 3.66066275854242e-20 0.17795582484125 4.72458028745174e-20 3.80676589318223e-20 3.66979116100265e-20 2.71318234348475e-20 0.0558529287112819 2.19355013036344e-20 0.0381587516493655 2.42417308498371e-20 4.71892355961059e-20 0.0723727663350696 5.64473945464854e-20 0.00290005566939826 6.96663918906589e-20 5.74590300026141e-20 4.25834079286928e-20 9.07006048511931e-20 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 9.07315459354617e-20 0.113740033328324 6.10513091937558e-20 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 0.14662301932474 9.14098254498316e-20 5.82597787560824e-20 4.80460431933668e-20 3.85987478930723e-20 3.26120035757344e-20 3.61128385116762e-20 4.81650909640983e-20 0.212188986318527 0.00288790109573657 2.90817268902941e-20 3.57892404846891e-20 0.0947007289909687 3.1895005854227e-20 0.000726957644378531 3.74763307217287e-20 3.10815414898578e-20 4.41061706196116e-20 0.0150948598842241 2.94386643348713e-20 0.0123827208615362 0.00601257305742931 0.00249255171453374 2.3589125068214e-20 +clustered_tds_1-10Kb 0 0 0 0 0 0 0.00730103666481133 1.48334001522991e-19 0.0147056718567668 2.18598846219008e-19 1.84029395221006e-19 1.73644761214785e-20 1.49696579630446e-20 3.47114419294204e-20 2.53995730561859e-20 0.012734733235319 0.00255341092685728 2.09258714259033e-20 2.55596049694926e-20 0.000139912232541061 0.00877140069573296 0.00467014880681445 0.0017098881951034 2.20467501238808e-20 1.60488712304164e-10 0.00166113570785634 8.52361933168883e-13 5.50224819201509e-06 2.80063500983306e-19 1.56314289955482e-19 1.47092264292995e-19 2.48993649411402e-17 0.0182992538672415 1.92024108155225e-05 0.000951692672242778 0.00126909170946745 0.00821625416038658 0.00802654614822077 2.37367888686499e-06 7.55364316665385e-20 0.0138411176748775 0.00118071868193175 0.00842954842815933 7.50149635663954e-20 3.51496069342126e-15 0.00302274149239397 1.2289108304478e-19 0.00761513973895149 0.00954328142683056 1.58929435340401e-20 2.2439059470699e-20 2.20179740219242e-20 3.19927602094556e-20 0.0100245147188553 0.00300332378881603 3.64885180691663e-20 0.000683139877540607 4.50470189091067e-20 0.00291506827856146 5.59117980134354e-20 8.85143662169646e-20 3.1201016757648e-20 3.95236128409755e-20 4.43726306848287e-20 3.66066275854242e-20 3.8645359827963e-20 0.00686324282190191 0.00464806176483151 3.66979116100265e-20 2.71318234348475e-20 0.00541003195234853 2.19355013036344e-20 0.00869687559556418 2.42417308498371e-20 4.71892355961059e-20 0.00114610111457386 0.000524018753047078 6.82065848159122e-20 6.96663918906589e-20 5.74590300026141e-20 0.00155772703187894 9.07006048511931e-20 0.000355509968304541 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 0.00412618625000383 0.010762419956581 6.10513091937558e-20 0.000543669129156783 7.39595841870445e-20 0.00237534441896493 6.84671427066716e-20 9.14098254498316e-20 5.82597787560824e-20 0.00998048075591312 3.85987478930723e-20 0.00499578044540315 0.00984350546272477 0.00991697839326525 3.53569749313383e-20 3.33025645875325e-20 2.90817268902941e-20 3.57892404846891e-20 4.50620579958517e-20 3.1895005854227e-20 0.000396478289964603 0.00432337645270999 0.000458509333532557 0.0149803855100383 4.40547697474683e-20 0.000121446695246504 0.0110768778158818 3.87919548021642e-20 4.24615471433568e-20 2.3589125068214e-20 +clustered_tds_10-100Kb 1.13644702532971e-19 0.00790848580400985 1.02780470407957e-19 1.22619698724958e-19 1.28613540869706e-19 1.10123020060586e-19 1.8733134051656e-19 0.00730586238589032 1.77069036322592e-19 0.0161223567989728 1.84029395221006e-19 1.48061837658734e-12 0.00129083933339898 3.47114419294204e-20 0.00592701396314912 0.00486861514459533 0.00184309926926874 2.06229284563106e-12 0.00482747984842451 0.00518661318871394 0.011453191247698 0.00322520080932207 1.77712011350644e-20 2.20467501238808e-20 1.55467333841473e-20 0.000846084023855711 0.00854034396426614 0.0296312441606727 0.00465473662225979 0.0124297771272621 1.47092264292995e-19 0.000358440714145386 0.0174979404754965 5.38720357252812e-20 5.90949151544599e-20 0.000485409042742245 5.54797685328391e-20 0.0203333260390287 0.0105609499906239 7.55364316665385e-20 9.38762695860971e-20 7.36136279388596e-20 0.0119429030725658 7.50149635663954e-20 0.0168076585172986 1.05461989671689e-19 0.00335308416274063 1.04187557790194e-05 0.0109195182511344 1.58929435340401e-20 2.2439059470699e-20 0.00787899705806887 3.19927602094556e-20 0.0319462973319811 0.000236909048112906 3.64885180691663e-20 0.0063036381902221 4.50470189091067e-20 0.00295741869088688 5.59117980134354e-20 3.07572841045254e-20 3.1201016757648e-20 0.0270328769710012 4.43726306848287e-20 3.66066275854242e-20 0.00344302644049094 0.0131051067339521 0.00567860075005847 3.66979116100265e-20 2.71318234348475e-20 3.39456382921865e-20 0.00148815098087355 0.0210811771350969 2.42417308498371e-20 4.71892355961059e-20 0.00153522219229728 0.0107029135704859 0.00851883375512979 0.00223594334949822 0.000598750693514119 0.00114325762262581 9.07006048511931e-20 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 0.0179269723290916 0.00507502760366604 6.10513091937558e-20 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 6.84671427066716e-20 9.14098254498316e-20 5.82597787560824e-20 0.0209345808863425 0.00264180898608683 0.00150412106686393 3.61128385116762e-20 0.0184314744484552 0.00576291431437437 0.00866232125806817 2.90817268902941e-20 0.00237395564531466 4.50620579958517e-20 3.1895005854227e-20 0.00444773342566551 0.0135391269375427 0.00298469687412223 0.00489731382437148 0.00684014801630097 2.94386643348713e-20 0.0153170100631361 0.00448848971624071 4.24615471433568e-20 0.00256821412801387 +clustered_tds_100Kb-1Mb 0.00592419389057704 8.96489862652724e-20 0.00977411262771796 1.22619698724958e-19 0.0247777985291581 0.00687659023939403 0.00228323951437313 1.48334001522991e-19 0.00572580963442048 0.0124961573159856 0.0224519485162069 1.23338556148178e-13 0.000965154002624121 3.47114419294204e-20 0.00532005922478487 0.0328495081184733 0.00572679378349604 2.09258714259033e-20 0.0216689039654117 0.0111608611929747 0.0400729907962858 2.82643086695692e-20 1.77712011350644e-20 0.00732111383263299 1.55467333841473e-20 1.4436164942841e-20 0.00264512131722328 2.26501583960149e-19 2.80063500983306e-19 0.0287069693771404 1.47092264292995e-19 1.71132737562608e-19 0.12440636449716 0.0076751647586867 5.90949151544599e-20 4.91432899251023e-20 0.017394420547225 0.0410982417386384 0.0633015860895802 7.55364316665385e-20 9.38762695860971e-20 7.36136279388596e-20 0.000506835026723623 7.50149635663954e-20 0.02420302826491 1.05461989671689e-19 1.2289108304478e-19 2.25756462441701e-20 0.0380967685962999 1.58929435340401e-20 2.2439059470699e-20 0.00263534175621365 3.19927602094556e-20 0.0439590776958147 0.00125660599911461 3.64885180691663e-20 4.3921818528298e-20 0.00167006732332359 0.0106542686293131 3.14997352511135e-13 0.00477891517124773 3.1201016757648e-20 0.00291086945534368 0.0075021465535912 1.48269557119508e-15 0.0174975090840881 0.0425532064392714 0.0160947324649598 3.66979116100265e-20 2.71318234348475e-20 0.00663069314961285 0.00923702231490937 0.0325590142018725 2.42417308498371e-20 4.71892355961059e-20 0.0114541318282961 0.00821222378115348 0.0339270441896373 0.00307397218166383 5.74590300026141e-20 4.25834079286928e-20 0.00394273580888227 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 0.065653136845419 8.45408369612758e-20 0.0350864451523768 4.19620688953765e-20 7.39595841870445e-20 0.0136156826528976 6.84671427066716e-20 9.14098254498316e-20 5.82597787560824e-20 1.35087110758569e-15 3.85987478930723e-20 0.0375256906470529 3.61128385116762e-20 0.0182607749470684 0.016216579490401 0.00789649875550715 2.90817268902941e-20 0.000299740689490781 4.50620579958517e-20 3.1895005854227e-20 0.029856332868627 3.74763307217287e-20 3.10815414898578e-20 0.0509258532490647 0.0170822118175198 0.00132477319421523 0.0450644259350711 3.87919548021642e-20 0.00487749490275899 5.09574036974805e-06 +clustered_tds_1Mb-10Mb 0.0286474288795963 8.96489862652724e-20 1.02780470407957e-19 1.22619698724958e-19 0.118342323442449 0.0335908204410253 0.0425268260078487 1.48334001522991e-19 1.77069036322592e-19 0.00692972996747551 0.0491844160855363 0.023847184630029 1.49696579630446e-20 3.47114419294204e-20 0.00178554984856099 0.168123452376716 0.000265576190290521 0.00172124807956887 2.55596049694926e-20 0.00179174633755428 0.118277984630983 2.82643086695692e-20 1.77712011350644e-20 0.02568915109001 1.55467333841473e-20 1.4436164942841e-20 2.65954193058143e-14 2.26501583960149e-19 0.0278005338718382 0.095087611086963 1.47092264292995e-19 0.0612212430506644 0.0287366911623741 0.00546545643213387 0.00222082475918246 4.91432899251023e-20 0.0623563501964706 1.18951731027594e-13 0.0570448383437894 0.00653547210093 0.00491367814947953 2.41500801299274e-11 0.0491377874404454 0.00150789030643165 0.036304542397365 1.05461989671689e-19 1.2289108304478e-19 2.25756462441701e-20 0.109012360097064 0.0217594848608093 2.2439059470699e-20 0.0464220758920141 3.19927602094556e-20 0.0291787014021147 0.00321999441106253 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 0.0918939971686957 5.59117980134354e-20 3.07572841045254e-20 0.00113321300817757 3.95236128409755e-20 4.43726306848287e-20 3.66066275854242e-20 0.0239302155877112 0.19662118700333 0.0174327362369955 3.66979116100265e-20 2.71318234348475e-20 0.0339018319449111 0.00128501807626077 0.0474858976493946 2.42417308498371e-20 4.71892355961059e-20 0.0761349129229563 0.0016464500245484 0.0209033520979248 0.00103714814745112 5.74590300026141e-20 4.25834079286928e-20 9.07006048511931e-20 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 0.0529698297237705 0.0662421428598166 0.0921361654595861 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 0.0456860474294261 9.14098254498316e-20 5.82597787560824e-20 4.80460431933668e-20 3.85987478930723e-20 0.169133045732187 3.61128385116762e-20 4.81650909640983e-20 0.00447631157676343 0.000165114267159425 0.00063044240288123 3.57892404846891e-20 0.0218165077140392 3.1895005854227e-20 0.000301241998632165 3.74763307217287e-20 3.10815414898578e-20 0.119877039419681 4.40547697474683e-20 2.94386643348713e-20 0.0116043481880667 3.87919548021642e-20 4.24615471433568e-20 2.3589125068214e-20 +clustered_tds_>10Mb 1.13644702532971e-19 8.96489862652724e-20 1.02780470407957e-19 1.22619698724958e-19 1.28613540869706e-19 0.166404143429094 1.8733134051656e-19 1.48334001522991e-19 1.77069036322592e-19 2.18598846219008e-19 0.048309292146005 1.73644761214785e-20 0.00471519866411483 3.47114419294204e-20 2.53995730561859e-20 1.67991834076044e-20 0.215216410421293 2.09258714259033e-20 2.55596049694926e-20 2.23326137118201e-20 1.81011962568141e-06 2.82643086695692e-20 1.77712011350644e-20 0.193038336623886 1.55467333841473e-20 1.4436164942841e-20 0.00138955460159977 2.26501583960149e-19 0.00877911595952787 1.56314289955482e-19 1.47092264292995e-19 0.133374651169745 1.54196727801832e-19 5.38720357252812e-20 5.90949151544599e-20 4.91432899251023e-20 0.0973629694738498 7.08490153865265e-20 8.23014850574395e-20 7.55364316665385e-20 0.00615578304701311 7.36136279388596e-20 0.116508694055318 7.50149635663954e-20 0.0208414965614503 1.05461989671689e-19 1.2289108304478e-19 2.25756462441701e-20 0.0139775979994241 1.58929435340401e-20 2.2439059470699e-20 0.166741417198212 3.19927602094556e-20 5.41480440732341e-20 4.1974369108772e-20 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 0.122380938440581 5.59117980134354e-20 3.07572841045254e-20 3.1201016757648e-20 3.95236128409755e-20 4.43726306848287e-20 3.66066275854242e-20 0.194946697204399 0.00834833323383674 3.80676589318223e-20 3.66979116100265e-20 2.71318234348475e-20 0.0388404562273218 0.00203770557782076 0.0393358162033824 2.42417308498371e-20 4.71892355961059e-20 0.0766935285043275 5.64473945464854e-20 6.82065848159122e-20 6.96663918906589e-20 5.74590300026141e-20 4.25834079286928e-20 9.07006048511931e-20 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 9.07315459354617e-20 0.119349130862324 6.10513091937558e-20 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 0.154094256009812 9.14098254498316e-20 5.82597787560824e-20 4.80460431933668e-20 3.85987478930723e-20 3.26120035757344e-20 3.61128385116762e-20 4.81650909640983e-20 0.191315502475618 0.00888415198029799 2.90817268902941e-20 3.57892404846891e-20 0.149058635786283 3.1895005854227e-20 2.8220400790683e-20 3.74763307217287e-20 3.10815414898578e-20 4.41061706196116e-20 4.40547697474683e-20 2.94386643348713e-20 0.0173233312811372 3.87919548021642e-20 4.24615471433568e-20 2.3589125068214e-20 +clustered_inv_1-10Kb 3.69668899181415e-15 8.96489862652724e-20 1.02780470407957e-19 1.22619698724958e-19 0.0840932170001299 1.10123020060586e-19 0.040731535458113 0.00372732112136939 1.77069036322592e-19 2.18598846219008e-19 0.0254680212012878 1.73644761214785e-20 0.0135759566552698 3.47114419294204e-20 0.0340447863935963 0.0292643269154893 0.00755971976641311 0.00176528750396511 0.0117772651735987 2.23326137118201e-20 0.0369062648242223 0.0442771804738051 0.00194222421908105 2.20467501238808e-20 1.55467333841473e-20 1.4436164942841e-20 0.0257981638023001 0.0649045429441039 0.00710357489363758 0.00313193069839871 1.47092264292995e-19 1.71132737562608e-19 0.0373824961291246 5.38720357252812e-20 5.90949151544599e-20 0.00424337217138766 0.0315192581918193 0.000507481332940068 0.0695999344485632 0.0104836549198934 0.00247003212838957 7.36136279388596e-20 9.86620112378054e-05 0.0319710437240117 0.0750230441786925 1.05461989671689e-19 0.00932229561407753 2.25756462441701e-20 0.037707146861504 1.58929435340401e-20 0.0028231724322052 0.0164939016120841 3.19927602094556e-20 0.0634666969186082 0.000668027190568128 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 0.00628429300872541 5.59117980134354e-20 3.07572841045254e-20 3.1201016757648e-20 3.95236128409755e-20 0.197398775501576 3.66066275854242e-20 3.8645359827963e-20 4.72458028745174e-20 5.83278624975959e-09 3.66979116100265e-20 2.71318234348475e-20 3.39456382921865e-20 0.00168150755695962 0.0868181644733378 2.42417308498371e-20 4.71892355961059e-20 0.0217472991747212 5.64473945464854e-20 0.0768887108337424 6.96663918906589e-20 5.74590300026141e-20 4.25834079286928e-20 9.07006048511931e-20 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 0.125411283661419 8.45408369612758e-20 6.89101335578525e-05 4.19620688953765e-20 0.00252646622269735 7.89774967501189e-20 0.0241066989252349 9.14098254498316e-20 5.82597787560824e-20 0.0122835315309236 0.00552466162411002 0.0377238657196725 0.0228693674256127 0.0109950644740006 0.0130067142480883 0.00948379712297617 2.90817268902941e-20 3.57892404846891e-20 0.0363720595593452 0.000779236728594356 0.00641065724982919 3.74763307217287e-20 3.10815414898578e-20 0.0551738426336551 0.0354683091483915 2.94386643348713e-20 0.0422350573945847 0.0117035355480028 4.24615471433568e-20 0.00738957466864555 +clustered_inv_10-100Kb 1.13644702532971e-19 8.96489862652724e-20 1.02780470407957e-19 1.22619698724958e-19 0.0212461036235714 1.10123020060586e-19 0.0129311953773923 0.0123658595889348 1.77069036322592e-19 2.18598846219008e-19 0.0310892050698084 1.73644761214785e-20 0.00960339094411555 3.47114419294204e-20 0.0149469946876881 0.00631969644412336 0.00855070169402491 2.09258714259033e-20 0.00571047657832403 1.78866751196356e-18 0.0321486845310465 0.00297160907052222 0.000875817372679354 2.53363467449075e-08 1.52427707067701e-11 1.4436164942841e-20 0.0132639310818568 0.00393244972970072 0.00653442900388887 0.0222632245411201 1.47092264292995e-19 1.71132737562608e-19 0.054028577201535 8.16379824856861e-11 5.90949151544599e-20 4.91432899251023e-20 0.0189184364635793 7.08490153865265e-20 0.0513364415628387 0.00512438722536105 9.38762695860971e-20 7.36136279388596e-20 8.86591200104389e-20 0.00922742777767563 0.0302537853311298 2.05704332307957e-13 0.00675111260022502 2.25756462441701e-20 0.0555518718362927 0.00513538022436023 2.2439059470699e-20 2.20179740219242e-20 3.19927602094556e-20 0.0355228958326864 1.41421938626612e-07 3.64885180691663e-20 0.00175200325002683 4.50470189091067e-20 0.00301796492229556 5.59117980134354e-20 3.07572841045254e-20 3.1201016757648e-20 3.95236128409755e-20 0.0740901900319891 3.66066275854242e-20 3.8645359827963e-20 0.00337044312023818 0.00483164818272403 3.66979116100265e-20 2.71318234348475e-20 0.0215438336405156 0.000525214160693543 0.0503122997292549 2.42417308498371e-20 4.71892355961059e-20 0.00631025821048007 0.00827367841177799 0.0274083741770639 6.96663918906589e-20 5.74590300026141e-20 4.25834079286928e-20 0.00301784230223323 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 0.00897313603660786 0.0494327540514997 8.45408369612758e-20 0.0111859187245403 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 0.0122675684855591 9.14098254498316e-20 5.82597787560824e-20 0.0206668252765942 0.00388691833791089 0.0201982720239042 3.61128385116762e-20 0.0115935360525007 0.0136614682916156 0.011941966044709 2.90817268902941e-20 3.57892404846891e-20 0.0114991208406889 3.1895005854227e-20 0.00502988328352087 3.74763307217287e-20 0.00549787914771319 0.0389288491462366 0.014966422447095 2.94386643348713e-20 0.0353405149363848 0.000672557840509334 4.24615471433568e-20 0.00177733779075533 +clustered_inv_100Kb-1Mb 0.0184874947106427 0.014892977849029 1.02780470407957e-19 1.22619698724958e-19 0.0383139144302804 1.10123020060586e-19 0.0201526681726685 0.00460838986499092 1.77069036322592e-19 2.18598846219008e-19 0.0257854889973456 0.00649740688199457 0.0111032938233231 3.47114419294204e-20 0.0139034106754738 0.059248989170463 0.00861737152390362 0.00289315111344519 2.55596049694926e-20 2.23326137118201e-20 0.0956569729875166 2.82643086695692e-20 1.77712011350644e-20 0.0050256127584281 1.55467333841473e-20 1.4436164942841e-20 0.00202502516577692 2.26501583960149e-19 0.00308001697808629 0.025839014313547 1.47092264292995e-19 1.71132737562608e-19 0.220424314603946 5.38720357252812e-20 5.90949151544599e-20 0.00303807378186342 0.0366544706481848 7.08490153865265e-20 0.14005821862808 7.55364316665385e-20 9.38762695860971e-20 0.00581346336324719 4.52884877026111e-09 7.50149635663954e-20 0.0504229755518958 1.05461989671689e-19 1.2289108304478e-19 2.25756462441701e-20 0.064287833895912 1.40391643612548e-13 2.2439059470699e-20 0.019482249602294 3.19927602094556e-20 0.0682404390890975 0.00116517696178471 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 0.028337800064611 5.59117980134354e-20 3.07572841045254e-20 3.1201016757648e-20 3.95236128409755e-20 0.0536361979273396 3.66066275854242e-20 0.0101631480410074 0.107399397418534 0.0286006663318271 1.11661248423575e-09 2.71318234348475e-20 0.0101182716059287 0.00581272606479718 0.0832065540990284 2.42417308498371e-20 4.71892355961059e-20 0.0386290745937421 0.00472199767757048 0.0353537157460369 6.96663918906589e-20 5.74590300026141e-20 4.25834079286928e-20 9.07006048511931e-20 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 3.91111320176704e-16 0.0899241597275232 0.0291290243220229 0.104968061545623 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 0.0261875918464899 9.14098254498316e-20 5.82597787560824e-20 0.0200690285694628 3.85987478930723e-20 0.0831199645111926 3.61128385116762e-20 0.00271548562051638 0.00678431795930544 0.0148455593565254 8.80351928309474e-08 3.57892404846891e-20 4.50620579958517e-20 3.1895005854227e-20 0.0114688922328245 3.74763307217287e-20 0.00698992937720981 0.0842381900570174 0.0269041569728642 2.94386643348713e-20 0.0606671410914512 0.00145834766895399 0.00476664126623786 2.3589125068214e-20 +clustered_inv_1Mb-10Mb 0.0316253271649403 8.96489862652724e-20 1.02780470407957e-19 1.22619698724958e-19 0.199000823935906 0.074169108015155 0.0281284550546849 1.48334001522991e-19 1.77069036322592e-19 2.18598846219008e-19 0.0738900774563034 0.0305180308357512 0.0168192542607574 3.47114419294204e-20 2.53995730561859e-20 0.325147277617037 0.0149487473360524 2.09258714259033e-20 2.55596049694926e-20 2.23326137118201e-20 0.259833162477456 2.82643086695692e-20 1.77712011350644e-20 0.0553627883353532 1.55467333841473e-20 1.4436164942841e-20 8.7985858819092e-05 2.26501583960149e-19 0.0468219517841486 0.232881786096718 1.47092264292995e-19 0.0738422588224709 0.0308008903226252 0.0161805500899706 5.90949151544599e-20 4.91432899251023e-20 0.0976025322791356 7.08490153865265e-20 0.130634850834933 0.00227952472040438 0.0200069260633629 7.36136279388596e-20 0.0755277421316515 7.50149635663954e-20 0.0719367784540381 1.05461989671689e-19 1.2289108304478e-19 2.25756462441701e-20 0.190377006010296 0.0443113927036542 2.2439059470699e-20 0.119529865311352 3.19927602094556e-20 0.0591672291625437 4.1974369108772e-20 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 0.157052831054719 5.59117980134354e-20 3.07572841045254e-20 3.1201016757648e-20 3.95236128409755e-20 4.43726306848287e-20 3.66066275854242e-20 0.0525047907803151 0.384618540515757 2.85566848929026e-09 3.66979116100265e-20 2.71318234348475e-20 0.0608107595954085 2.19355013036344e-20 0.15005301179945 2.42417308498371e-20 4.71892355961059e-20 0.117784437912795 5.64473945464854e-20 0.0161392029673007 0.00107265552731405 5.74590300026141e-20 4.25834079286928e-20 9.07006048511931e-20 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 0.11910627138356 0.124889818305552 0.200117813182694 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 0.0265248074623661 9.14098254498316e-20 5.82597787560824e-20 8.3207394908559e-17 3.85987478930723e-20 0.31047845693447 3.61128385116762e-20 4.81650909640983e-20 0.0446527419542065 0.0149868886902672 2.90817268902941e-20 3.57892404846891e-20 0.0775488337916435 3.1895005854227e-20 2.8220400790683e-20 3.74763307217287e-20 3.10815414898578e-20 0.221734068546583 4.40547697474683e-20 2.94386643348713e-20 0.033853443006377 0.00905907053646683 4.24615471433568e-20 2.3589125068214e-20 +clustered_inv_>10Mb 0.0123949769177571 8.96489862652724e-20 1.02780470407957e-19 1.22619698724958e-19 0.0418859464290602 0.385735166178544 1.8733134051656e-19 0.0163197861950655 1.77069036322592e-19 0.0416091038851855 0.0790894001058383 8.14891645692392e-13 0.00232442850561754 3.47114419294204e-20 2.53995730561859e-20 0.0191604209712535 0.420830460043783 2.09258714259033e-20 2.55596049694926e-20 2.23326137118201e-20 1.87723865406083e-20 2.82643086695692e-20 1.77712011350644e-20 0.391344888840297 1.55467333841473e-20 1.4436164942841e-20 1.90694174609194e-20 2.26501583960149e-19 0.00731592996627322 1.56314289955482e-19 1.47092264292995e-19 0.309151648087098 1.54196727801832e-19 5.38720357252812e-20 5.90949151544599e-20 4.91432899251023e-20 0.202190789704503 0.00386811666376444 8.23014850574395e-20 7.55364316665385e-20 9.38762695860971e-20 7.36136279388596e-20 0.242726445939298 7.50149635663954e-20 0.0457168311670522 1.05461989671689e-19 1.2289108304478e-19 2.25756462441701e-20 2.86083723118323e-20 0.0260237841998593 2.2439059470699e-20 0.308549764628523 3.19927602094556e-20 5.41480440732341e-20 4.1974369108772e-20 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 0.229615721133565 5.59117980134354e-20 3.07572841045254e-20 3.1201016757648e-20 3.95236128409755e-20 4.43726306848287e-20 3.66066275854242e-20 0.325564139994419 4.72458028745174e-20 3.80676589318223e-20 3.66979116100265e-20 2.71318234348475e-20 0.115223868581468 0.000545523664127745 0.0606810569698098 2.42417308498371e-20 0.000805531645134193 0.14258515158551 5.64473945464854e-20 6.82065848159122e-20 6.96663918906589e-20 5.74590300026141e-20 4.25834079286928e-20 9.07006048511931e-20 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 9.07315459354617e-20 0.25240938902998 6.10513091937558e-20 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 0.326399651366238 9.14098254498316e-20 5.82597787560824e-20 4.80460431933668e-20 3.85987478930723e-20 2.85587937465129e-16 3.61128385116762e-20 3.00045944545309e-09 0.363289443826533 0.0131148444668847 2.90817268902941e-20 3.57892404846891e-20 0.206032337906043 3.1895005854227e-20 2.8220400790683e-20 3.74763307217287e-20 3.10815414898578e-20 4.41061706196116e-20 4.40547697474683e-20 2.94386643348713e-20 0.0225022884112955 0.000777324655910749 4.24615471433568e-20 2.3589125068214e-20 +clustered_trans 0.563686301096126 8.96489862652724e-20 1.02780470407957e-19 1.22619698724958e-19 1.28613540869706e-19 1.10123020060586e-19 0.0401437554818816 1.48334001522991e-19 0.00868968481111955 2.18598846219008e-19 0.195985787667194 1.89471967209957e-14 0.87485425951719 3.47114419294204e-20 2.53995730561859e-20 3.34573681938244e-14 1.3110361577648e-20 2.54906817131489e-20 2.55596049694926e-20 2.23326137118201e-20 1.87723865406083e-20 2.82643086695692e-20 1.77712011350644e-20 2.20467501238808e-20 1.55467333841473e-20 1.4436164942841e-20 0.820622940804924 0.0122408123748752 0.017951762352658 0.322451979149426 1.47092264292995e-19 1.71132737562608e-19 1.54196727801832e-19 0.0213775970078916 5.90949151544599e-20 4.91432899251023e-20 0.0223976732617141 7.08490153865265e-20 8.23014850574395e-20 7.55364316665385e-20 0.547181762254774 7.36136279388596e-20 8.86591200104389e-20 7.50149635663954e-20 0.0907613559934125 1.05461989671689e-19 1.2289108304478e-19 2.25756462441701e-20 2.86083723118323e-20 0.834054172845563 2.2439059470699e-20 2.20179740219242e-20 3.19927602094556e-20 0.425952202721484 4.1974369108772e-20 3.64885180691663e-20 4.3921818528298e-20 4.50470189091067e-20 3.78655317241334e-20 5.59117980134354e-20 3.07572841045254e-20 3.1201016757648e-20 3.95236128409755e-20 4.43726306848287e-20 3.66066275854242e-20 3.8645359827963e-20 4.72458028745174e-20 0.604826622868524 3.66979116100265e-20 2.71318234348475e-20 0.444322910022656 2.19355013036344e-20 3.9862459500132e-20 2.42417308498371e-20 4.71892355961059e-20 7.71429335399417e-20 5.64473945464854e-20 0.224501880646808 0.00422381811182628 5.74590300026141e-20 4.25834079286928e-20 9.07006048511931e-20 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 0.806073294533329 9.07315459354617e-20 8.45408369612758e-20 6.10513091937558e-20 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 6.84671427066716e-20 9.14098254498316e-20 5.82597787560824e-20 0.602979850460582 3.85987478930723e-20 3.26120035757344e-20 3.61128385116762e-20 4.81650909640983e-20 3.53569749313383e-20 0.747267113450847 2.90817268902941e-20 3.57892404846891e-20 4.50620579958517e-20 3.1895005854227e-20 2.8220400790683e-20 3.74763307217287e-20 3.10815414898578e-20 4.41061706196116e-20 0.645023557149198 2.94386643348713e-20 0.120005777722113 4.91600784265811e-14 0.00386890567782029 2.78722205388542e-15 +non-clustered_del_1-10Kb 0.0115761161126879 8.96489862652724e-20 0.267551996983944 0.0271437290973461 7.84116985717084e-11 0.0281787026836645 0.0729169247805847 0.0440222462828933 0.138529176626333 0.141192122697707 2.23305561142058e-09 0.0165529299933585 0.00105166729328509 0.0103847890537415 0.158052509466118 1.67991834076044e-20 0.000473926560472093 0.00157418025131964 0.0127155409187183 5.66431982485622e-09 1.87723865406083e-20 7.59487971062102e-16 0.397668128477853 2.20467501238808e-20 1.55467333841473e-20 0.0352859967416336 1.90694174609194e-20 0.255975407751935 0.0617424645775362 0.0016287453171104 0.0712411865936868 0.0149293552667551 0.0359046395261191 0.0633485529051214 0.0655932041771128 0.149608775886979 3.05418744400478e-10 0.0189353175913863 0.0474107644543664 0.509941263021196 2.72222959728725e-19 0.0972730558544273 1.59526702016391e-13 7.50149635663954e-20 6.44643891971382e-15 0.284362975428482 0.0804701282304814 0.134062955933948 2.86083723118323e-20 0.00147179322154013 0.0424889613198103 0.00310646975715426 0.366166282081867 2.3410091388459e-16 0.00930835233005344 0.0105349210405757 5.88176650075674e-11 0.0042842664097484 0.00927034591249251 0.0585486840446503 0.0189160229079936 0.811900974664982 0.0792570591335994 1.9517571941099e-07 0.00179709680788925 0.013904090843934 1.44154543606741e-09 0.0165314615409071 0.0690649771743133 2.71318234348475e-20 3.39456382921865e-20 0.104074959776132 3.9862459500132e-20 0.316163603254809 0.426326087621451 0.0345736974629276 6.46256942253633e-08 3.42714810893515e-18 0.00269173185019707 0.00598498990060321 0.0848583323432471 9.07006048511931e-20 9.26847697415676e-20 0.581733430042147 7.08597427669922e-20 1.45477090902705e-19 9.07315459354617e-20 8.45408369612758e-20 6.10513091937558e-20 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 6.84671427066716e-20 9.14098254498316e-20 0.943078392244418 4.80460431933668e-20 3.85987478930723e-20 0.0305102372464213 3.61128385116762e-20 0.00511701695397395 0.0144390168010095 3.33025645875325e-20 0.0369387431545071 0.316178474617439 0.0619295830430498 3.1895005854227e-20 1.43615006477211e-08 0.000822209436036316 0.0757305919572367 4.41061706196116e-20 8.35469724282685e-20 0.537323727508422 2.99776934902128e-20 0.189526299565431 0.00576998780095724 1.06740289717184e-08 +non-clustered_del_10-100Kb 0.00375113714136821 0.152398688454613 0.107742420433436 1.22619698724958e-19 0.0105770322384533 0.00726259113184628 1.21944926986646e-14 0.011115861832354 0.0816221262994578 0.159799676755232 0.0258823890895872 0.0219105636856052 1.49696579630446e-20 0.0140954305153824 0.205389413480754 1.12901217100235e-07 1.3110361577648e-20 0.011332132617993 0.0857788574604667 0.0108429406970319 1.87723865406083e-20 2.82643086695692e-20 0.248349751155337 2.20467501238808e-20 5.60060921752347e-07 0.00246609487570341 1.06227758205703e-05 0.103471585995694 0.0501702231693559 0.00611660474719724 0.023430776009811 0.0105635738528823 0.0295237511371344 0.0315720951268279 0.0526181101818984 0.296091829238488 0.00261295318469242 0.0257378240001785 0.0203184883241809 0.0512542896976429 0.0246726969061444 0.318669413301855 0.036694942721163 0.0130654278574193 0.0767146705169684 0.183477739888515 1.66082504955348e-13 0.297578080948145 2.86083723118323e-20 0.00240803691240866 2.2439059470699e-20 1.48926812803203e-14 0.288116972499225 0.015102540408871 0.00833421556308343 3.64885180691663e-20 0.0205081228995956 0.00165202025580415 0.00721324582263335 5.59117980134354e-20 6.42316434950944e-20 0.0531438579356497 0.050154434303489 4.43726306848287e-20 0.0319599637118949 3.8645359827963e-20 4.72458028745174e-20 0.008735140699616 0.403978582289422 0.0115796000397003 0.00662509209014459 0.046848691405216 3.9862459500132e-20 0.298542121391729 0.175423468925208 0.0220449454387433 0.00708787704775276 6.89426726417912e-08 9.85868455087944e-05 0.188441669421524 4.25834079286928e-20 0.00451133307923503 0.124670710238408 0.224058190229705 7.08597427669922e-20 1.45477090902705e-19 0.0059236511111185 8.45408369612758e-20 0.07772713114301 4.19620688953765e-20 0.00494336920944943 0.0172252733476629 1.52129280166065e-10 0.0036293153094393 0.0280149686666631 4.80460431933668e-20 0.173696617279646 3.26120035757344e-20 0.00024639363112569 0.00330662086651422 0.0110824857466258 0.00872840985659636 0.00255128718563744 0.255013138912592 2.004730785498e-07 8.13662008280182e-14 0.0180172047762424 0.0184136056450079 0.329357452140302 2.89548371214621e-09 5.90851357419842e-18 0.0566670137867892 1.24154609137142e-07 0.131545917437431 2.73515594347605e-07 0.02398890570178 +non-clustered_del_100Kb-1Mb 0.00272526956386616 0.163046547385735 0.120357338465236 0.0181838912346168 0.0396887252440441 0.00828923683980797 1.8733134051656e-19 0.0691746452194183 0.105869128964726 0.314812228377219 2.93520169805864e-12 0.0234655002676735 1.49696579630446e-20 3.47114419294204e-20 0.267705666454017 0.0151693548016936 0.000618428389017112 0.00877138856616144 0.0718908315476883 0.0267480056435584 0.0145621974937092 2.03573480324689e-09 0.0828364917930943 0.00277859703835002 0.0455118514311697 2.38119898863677e-13 3.62673709367522e-13 0.0550214004535066 0.0917098163267888 0.00430649205586907 0.0474738265509293 0.00389140204119685 0.0457668976769999 0.109474647423412 0.0315537609314308 0.274835703065359 0.0254819889194555 0.0309452561432949 8.23014850574395e-20 0.0380777369847793 0.0107202880351844 0.306604910263161 0.0663102948027998 0.0646992797220333 0.0711250203982612 0.13964304973495 2.91734146368513e-10 0.283872584894499 0.0353025951277793 1.58929435340401e-20 1.48649269652332e-16 2.20179740219242e-20 0.234990950577339 0.0173453938067101 0.0212358207810929 0.0294706269291809 0.0283795788230457 4.50470189091067e-20 0.00203868856636768 0.00390177238028613 0.00467791025638352 3.1201016757648e-20 5.84136177341817e-16 0.0870885810449704 3.66066275854242e-20 3.8645359827963e-20 0.0078674594913951 3.80676589318223e-20 0.379397348195981 0.136885540275195 0.0165830700761529 0.0124518459121233 0.0233092518659988 0.133758472772658 1.79329377661603e-15 0.00747059467598042 0.0345624852886353 0.00269576109415479 6.96663918906589e-20 0.245857412765047 1.546698996859e-17 9.07006048511931e-20 0.421235745127561 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 1.44616559086336e-19 8.45408369612758e-20 0.0097579371412851 0.0682391090926316 0.0194783062880062 0.0151031406836521 0.0360916767961185 9.14098254498316e-20 0.00842997973774147 0.00867736413971523 0.100202680803968 0.0117593980476301 0.0423894280385318 0.0532706570463285 4.01409027892945e-19 0.0183531088282914 0.00943117124887521 0.169026823826067 0.0138934387471537 0.0279736006295287 0.0268890433532982 3.74763307217287e-20 0.35428980901516 0.0481065087188732 0.0229485337228135 0.0111839715971124 0.0348013558716177 0.14240600002685 6.35130945880001e-16 0.0268703861200021 +non-clustered_del_1Mb-10Mb 1.13644702532971e-19 0.0467009386302167 0.0232734277464952 0.0197342425618444 1.28613540869706e-19 0.0187434201942456 0.0192242473625565 0.040429445270207 0.0353449019329645 0.0581343096939913 0.0316663808414637 0.10370271663506 0.000595751561461344 3.47114419294204e-20 0.0219341916781857 0.0126157850278417 0.00130870498280046 0.0112269924477692 0.00247543376310238 0.00936761086584704 0.0386668674936949 0.00509813374604451 0.00577187996468946 0.00301093141030617 0.0652340091834369 0.001147618279376 0.00216353779326796 0.0059568456995687 0.0473882330740344 0.00837305631628964 0.0425511304984514 0.0189675574287299 0.0261160068667628 0.0443712114194066 0.000697731310602841 0.00714503897073733 0.0146393606094493 7.08490153865265e-20 0.0327120190266462 1.39929793268425e-18 0.0238239712437578 0.0205057573909416 8.86591200104389e-20 0.0531543682735194 0.00143173319998051 0.00852721975678306 0.0576836820717819 0.0138722807465039 0.0177298540715793 0.00395717443419998 0.0673266880338227 0.00683495070349715 0.015197324584012 0.00994511057773265 0.0119443677845111 0.0504552892192745 4.3921818528298e-20 4.50470189091067e-20 0.0143093318775971 5.59117980134354e-20 0.0111219449373757 0.0135507515645656 3.95236128409755e-20 5.98247384580751e-11 0.0986727980462792 0.0142759742916317 0.0205700688174439 3.80676589318223e-20 0.0112872616823098 0.0703893037250106 3.39456382921865e-20 0.0135443600904974 0.0260568636997036 0.00946564018893832 0.00195156672630481 0.0329408268966284 9.39671941322156e-15 0.0175384722064317 0.0138940829263531 0.0512386621908816 0.00605013504667352 9.07006048511931e-20 0.103579387433512 0.00475126436123203 0.0550450199823431 1.45477090902705e-19 0.00135780346547316 0.0329370459138711 0.0311749370115175 0.0682344739915705 7.39595841870445e-20 7.89774967501189e-20 0.0218299040095326 3.73533217126923e-16 0.00479918643329055 0.0515222253495055 0.111848014990088 0.00311602763334577 0.122289856755776 0.00578512217476449 3.53569749313383e-20 0.00687372917982995 0.0159332326828768 0.0177438613040143 0.0193170271539778 0.0591230521247149 2.8220400790683e-20 0.00853396080343903 0.0206410703989331 0.0115121203821098 4.40547697474683e-20 0.035942060704624 0.0128726581939081 0.0389944647013581 0.00556518692838874 2.3589125068214e-20 +non-clustered_del_>10Mb 0.0193541854619674 0.0459874191343307 0.0296354379347337 0.0185961920801674 0.0394782303640145 0.0204892535093926 0.0222189897837327 0.0679620408310384 0.0114511004955133 2.18598846219008e-19 0.034222102002537 0.0738489948031729 3.43069860102065e-11 3.47114419294204e-20 0.00813242441700063 9.36024025661168e-11 0.0192247971761633 0.0230200480331857 0.000471789130484291 2.23326137118201e-20 8.96017209085951e-16 6.23973575098339e-06 0.00982029270520133 0.0302837981920173 0.0785014230559982 0.00667974181720459 1.90694174609194e-20 0.0192111563096015 0.00928821990995289 0.0140482405477647 0.0239392924045729 0.0355888066919044 1.54196727801832e-19 0.0518672584617655 0.000933949810686814 4.91432899251023e-20 0.0188499488912943 7.08490153865265e-20 8.23014850574395e-20 7.10910054373519e-14 0.0131394776411588 0.00920017817837881 0.0396559162855316 0.0545265811491627 0.00346622129959261 1.05461989671689e-19 0.0356926035577902 0.0384197661453318 2.86083723118323e-20 1.58929435340401e-20 2.2439059470699e-20 0.00975729014421025 0.0171492919931456 0.00884179397403825 4.1974369108772e-20 0.0719148031047641 0.00420128896960694 0.00307365528165109 0.0151932753661679 0.00675349442599067 0.0151324657839026 3.1201016757648e-20 0.00343046898585555 0.0441835151055778 0.0989969817392006 0.0168783179099858 4.72458028745174e-20 0.0145111437837984 7.23444037244069e-13 0.0711360219675933 0.00731134864234588 0.00633537113993804 0.0371640274167534 0.0093350173988778 0.0357557225552043 0.015573464451 5.64473945464854e-20 0.00901520190975095 0.0128683080939955 0.0216994808326168 0.00891978718029855 1.5796069698041e-11 0.0493713569267708 0.00890128407169123 0.0583232210424942 0.009167907368602 1.29792955536837e-11 0.0317149795082382 0.00618364244715189 0.0267711207529712 1.24788496597508e-19 7.89774967501189e-20 0.0489922865880416 0.00033390361916604 5.82597787560824e-20 4.80460431933668e-20 0.105271314774849 3.26120035757344e-20 0.0887088170286112 0.0108957445986364 3.53569749313383e-20 0.00998280474654287 0.0188038962282396 0.00786112738314711 4.50620579958517e-20 0.0675093123158158 0.0159984777387251 0.0115049115154649 0.0024138818344046 1.25731978535302e-07 0.0219248824218767 0.00951860596829949 0.00226411023536914 0.0241037253217911 0.00950521475899973 0.0161437560508585 +non-clustered_tds_1-10Kb 0.0119959461127507 0.00572255818468469 0.0118829343078482 1.22619698724958e-19 1.28613540869706e-19 2.58099276469959e-05 1.8733134051656e-19 1.48334001522991e-19 0.0802519408491384 0.00961507609624215 0.0140655117208154 1.73644761214785e-20 1.49696579630446e-20 3.47114419294204e-20 0.0734709000430056 1.67991834076044e-20 1.3110361577648e-20 2.09258714259033e-20 0.00305030026266666 2.23326137118201e-20 1.87723865406083e-20 2.82643086695692e-20 1.77712011350644e-20 2.20467501238808e-20 1.55467333841473e-20 0.484976254327758 1.90694174609194e-20 0.00449987102594485 0.0492252978990448 1.56314289955482e-19 0.0394816232972469 0.00266143125382229 0.00103553754146963 5.38720357252812e-20 0.187401431996632 0.0415478139314576 0.00671821697889331 0.0455992496635893 8.23014850574395e-20 0.00714043681945292 0.0288388326951505 0.050916074630442 0.0110163997997747 1.11003376145094e-07 0.0193172460680269 0.0642835623532659 0.00672257616979412 0.0112113388684493 0.0204789276209464 1.58929435340401e-20 0.0410728246366013 2.20179740219242e-20 0.00788960468045679 0.00480185463201667 4.1974369108772e-20 3.64885180691663e-20 0.168399650619399 4.50470189091067e-20 3.78655317241334e-20 5.59117980134354e-20 3.07572841045254e-20 9.59400374975691e-14 0.302188886739655 4.43726306848287e-20 3.66066275854242e-20 3.8645359827963e-20 4.72458028745174e-20 3.80676589318223e-20 3.66979116100265e-20 0.0607382146164443 3.39456382921865e-20 0.0146805266084617 0.00372264833006018 9.17422622908563e-14 0.00512112402444341 7.71429335399417e-20 5.64473945464854e-20 6.82065848159122e-20 8.82001449078466e-18 5.74590300026141e-20 0.334450649001651 0.130050433946046 9.26847697415676e-20 0.0178241003484369 0.00685855440038373 1.45477090902705e-19 9.07315459354617e-20 8.45408369612758e-20 6.10513091937558e-20 4.19620688953765e-20 7.39595841870445e-20 7.89774967501189e-20 6.84671427066716e-20 0.378679631782298 5.82597787560824e-20 4.80460431933668e-20 3.85987478930723e-20 0.0206524584978618 0.0170028315260677 0.0578657325818008 3.53569749313383e-20 3.33025645875325e-20 2.90817268902941e-20 0.0558069381757739 0.00639433490446567 3.1895005854227e-20 2.8220400790683e-20 0.159417486226027 0.00630704110035318 0.002782993775254 4.40547697474683e-20 2.94386643348713e-20 5.87465698604972e-10 0.0757562927040075 0.00548481471918221 0.010029831726125 +non-clustered_tds_10-100Kb 0.00740949839031924 0.141323960448695 0.00702842284785504 1.22619698724958e-19 0.026076444011828 1.10123020060586e-19 0.0313803558184726 0.00940887931014721 0.19463390332686 5.52759618129414e-11 0.0317526717795979 1.73644761214785e-20 1.49696579630446e-20 3.47114419294204e-20 0.0222039142829679 1.67991834076044e-20 1.3110361577648e-20 2.09258714259033e-20 0.368162068289124 0.176263285509921 1.87723865406083e-20 2.82643086695692e-20 0.0106092069256052 2.20467501238808e-20 1.55467333841473e-20 0.255083690481023 1.90694174609194e-20 0.0115894939057997 0.147620319874922 1.56314289955482e-19 0.0630885322635915 1.24790806928449e-13 1.54196727801832e-19 6.43953708387956e-11 0.421026071261144 7.14650051273173e-17 5.54797685328391e-20 0.22028091729807 1.1980957995194e-05 0.0322415746440271 0.0022472981010733 0.0755955883460071 1.82953484789073e-19 0.0349094406073137 0.0147833120952153 0.125836333853101 0.00197411178591246 0.0367718457649935 0.0157085990968137 1.33788678002682e-13 0.0571138329966827 0.00384879803458841 3.19927602094556e-20 5.41480440732341e-20 0.00881640693754834 3.64885180691663e-20 0.459392894196861 4.50470189091067e-20 3.78655317241334e-20 0.0443759922431796 0.0306748294552547 3.1201016757648e-20 0.439208408923299 4.43726306848287e-20 3.66066275854242e-20 0.00980986813159136 0.0277874130404439 0.0182541380451036 0.0241761304816574 2.71318234348475e-20 0.00443024922119217 0.182703793673251 3.9862459500132e-20 0.0366177291522637 4.71892355961059e-20 0.0308082501122899 0.284912206488389 6.82065848159122e-20 6.96663918906589e-20 8.55047754355134e-07 0.219713472585164 0.332264107534883 9.26847697415676e-20 0.0323800855153734 7.08597427669922e-20 1.45477090902705e-19 0.00251760481365721 9.9188043856902e-11 6.10513091937558e-20 4.19620688953765e-20 7.39595841870445e-20 0.042685707862118 6.84671427066716e-20 0.403966694997642 5.82597787560824e-20 4.80460431933668e-20 3.85987478930723e-20 3.26120035757344e-20 3.61128385116762e-20 0.330779749673328 3.53569749313383e-20 3.33025645875325e-20 2.90817268902941e-20 3.57892404846891e-20 4.50620579958517e-20 3.1895005854227e-20 0.101745142258264 0.267084746669553 3.39403665245068e-09 4.41061706196116e-20 0.0401232618012794 0.243014899180815 7.23890104008139e-07 0.107050952871641 0.0136252263924702 0.132124737740693 +non-clustered_tds_100Kb-1Mb 1.13644702532971e-19 0.069992044102221 0.0169288029635729 0.0348328016067847 1.28613540869706e-19 0.00251264239773529 0.1297030370014 1.17547910963913e-11 0.0400871297614177 0.0209640399877265 1.84029395221006e-19 0.000109959297353502 0.00157663834133554 3.47114419294204e-20 2.53995730561859e-20 1.67991834076044e-20 0.000573387883041238 0.000711906493224087 0.32769462472541 0.554490865388249 1.87723865406083e-20 2.82643086695692e-20 1.77712011350644e-20 2.20467501238808e-20 1.55467333841473e-20 0.00689682757178863 1.90694174609194e-20 2.26501583960149e-19 0.0760856716492415 1.56314289955482e-19 0.109238842811417 0.00672627991142715 0.00803300664184214 0.0398757678936459 0.036818609191812 3.91297594207242e-15 0.0146512944247778 0.366542442109967 1.39921597396356e-06 2.0096046773622e-17 0.00427853331724597 0.00239720278717521 8.86591200104389e-20 0.0228811306998159 0.0234637910771033 0.0547223146122951 0.0289635667871962 0.0178579896868426 0.0473916828656823 1.58929435340401e-20 0.0909015640079856 0.00328926276803727 3.19927602094556e-20 5.41480440732341e-20 0.645381535198406 3.64885180691663e-20 0.0745052261731696 4.50470189091067e-20 3.78655317241334e-20 0.718129727560632 3.07572841045254e-20 3.1201016757648e-20 3.95236128409755e-20 4.43726306848287e-20 3.66066275854242e-20 3.8645359827963e-20 4.72458028745174e-20 3.80676589318223e-20 3.66979116100265e-20 2.71318234348475e-20 3.39456382921865e-20 0.435715229788525 3.9862459500132e-20 2.42417308498371e-20 4.71892355961059e-20 7.71429335399417e-20 0.492286340682537 4.95929114163552e-08 0.371668343989304 5.74590300026141e-20 4.25834079286928e-20 0.374487383554324 9.26847697415676e-20 9.09254741825586e-20 7.08597427669922e-20 1.45477090902705e-19 9.07315459354617e-20 8.45408369612758e-20 6.10513091937558e-20 4.19620688953765e-20 7.39595841870445e-20 0.603703077619715 6.84671427066716e-20 9.14098254498316e-20 5.82597787560824e-20 4.80460431933668e-20 3.85987478930723e-20 3.26120035757344e-20 2.37682073302888e-18 0.295353508883274 0.00466439799504733 0.00140815718442364 0.0110636058607935 2.79621678293389e-10 0.0269677758490936 3.1895005854227e-20 0.513062543182671 0.217337268127379 3.10815414898578e-20 2.46296815762145e-10 0.0173372845514623 2.94386643348713e-20 0.158497806966006 3.10488377301427e-09 0.10032337418973 0.390987293952512 +non-clustered_tds_1Mb-10Mb 0.00487971599743002 0.0263824804313611 1.02780470407957e-19 1.22619698724958e-19 0.0114050024392991 1.10123020060586e-19 0.0268796151507609 0.028915058620749 0.0107306787372669 1.82426609203762e-08 0.0364609173771106 0.102993563769298 0.00661318622925733 3.47114419294204e-20 2.53995730561859e-20 0.00655212452506389 7.72289550434405e-07 0.00640065727592158 0.00845875995408234 0.144540327114804 0.0178551866948026 0.00922507634702375 7.3494533266459e-17 2.20467501238808e-20 0.0381750824917564 6.74839227100531e-15 7.66148122554992e-11 0.00748483738676144 0.00384249340591257 2.74710362815942e-10 0.0401538351166451 0.0169763561975702 0.0121802041434766 0.00961000851101176 0.00358917912769505 4.91432899251023e-20 0.0189178858902112 0.00887260679418155 0.0165231172418444 7.86570716451441e-19 0.0148021418288896 5.69094848190295e-19 0.00265813184714854 0.0470481004176016 0.00392990827077849 0.00556744892232754 0.0225203772969855 2.25756462441701e-20 0.0185205415931182 1.58929435340401e-20 0.0373985069473881 2.20179740219242e-20 3.19927602094556e-20 5.41480440732341e-20 0.227530874269721 1.580562649543e-17 4.3921818528298e-20 4.50470189091067e-20 0.0101531408030678 0.0600399893110672 0.0072792988067831 0.0145688250040694 0.00217642600232146 4.43726306848287e-20 0.0644172854698642 3.8645359827963e-20 0.0209197806393363 0.0106294509517769 0.0201683007949041 0.012060411263778 0.00249356329713041 0.0589309240866923 3.9862459500132e-20 2.42417308498371e-20 4.71892355961059e-20 0.00689654978283511 5.64473945464854e-20 6.82065848159122e-20 0.382786118291382 5.74590300026141e-20 4.25834079286928e-20 0.0354862704669862 0.0134044438504047 9.09254741825586e-20 0.0232887550161407 1.45477090902705e-19 0.023860987421978 0.0173038383712905 0.0189491489281278 0.0088861744233318 7.39595841870445e-20 0.207939011897078 0.015218993405381 9.14098254498316e-20 5.82597787560824e-20 0.00619970558834521 0.0105077327394177 0.000422936362854251 0.0475035752102593 0.0356989195938455 3.53569749313383e-20 0.0136415377110612 0.00949664305430171 3.57892404846891e-20 0.0292397113671388 0.00244641235684923 0.0973555647332045 3.74763307217287e-20 0.00311100193018166 4.41061706196116e-20 4.40547697474683e-20 2.94386643348713e-20 0.0574171349631062 0.00889077854773297 0.00953634604454111 0.0373578277159804 +non-clustered_tds_>10Mb 0.0250549401470152 2.6882139891034e-06 0.0244309316890965 0.051176692669531 0.0207945653499971 0.0363774744502198 1.8733134051656e-19 0.0485689844077963 0.000935315178275515 2.18598846219008e-19 1.84029395221006e-19 0.109364758222767 1.41470471507136e-05 3.47114419294204e-20 2.53995730561859e-20 1.67991834076044e-20 0.0127572861735087 0.0308649184756054 0.0142448138799179 0.000883436840525967 1.87723865406083e-20 2.82643086695692e-20 0.00681704385677512 0.0203312728005546 0.064456704549315 0.00829486812647564 3.63736784564052e-07 2.26501583960149e-19 0.0146318599325464 0.0002666367763192 0.0201779693208088 0.0144826952139649 1.54196727801832e-19 0.0260997610584779 0.00472929650284525 4.91432899251023e-20 0.0245952008328797 0.00401422031854327 8.23014850574395e-20 7.55364316665385e-20 0.00332974491293648 7.36136279388596e-20 0.0179846818584948 0.0586613960710625 0.00831760606564348 1.05461989671689e-19 0.0196140648407087 0.0260328558397986 0.00603268099741581 1.58929435340401e-20 0.0157775913950261 0.0139975495013968 0.00643177478064802 0.0194583437825658 0.003953137256771 0.0528826892572519 0.00426689636679234 4.50470189091067e-20 0.00870357622605833 0.0257952967696339 3.07572841045254e-20 9.4883096952029e-09 0.00205604763194346 4.50514701260348e-14 0.0938397706793839 0.00505302602790125 4.72458028745174e-20 0.00466455550377417 3.66979116100265e-20 0.0367055133954925 0.00810166930695588 0.000999599780495565 0.0248789498632196 0.00249602540339382 0.0231325890544038 0.0258885166477637 5.64473945464854e-20 0.005068581304578 0.0170806196367419 0.0161637900613323 0.0106604746116291 0.00128918501649415 0.0244484534300346 0.00654268745687704 0.0450313646170256 0.0150061877213556 0.00636878646786669 0.0161445191742398 6.10513091937558e-20 0.0358824883837623 7.39595841870445e-20 7.89774967501189e-20 0.0324814882356339 0.00523656475743699 0.00214559491467572 0.00900604266037364 0.0530727264569769 3.26120035757344e-20 0.0732435127434858 4.81650909640983e-20 0.00112633347796854 3.33025645875325e-20 0.0247098207128122 3.57892404846891e-20 0.0148850645196904 0.0348787881777666 0.00657721971988719 0.00405418962690642 3.10815414898578e-20 4.41061706196116e-20 9.44460980724176e-16 0.00751155131717398 0.00676036493330627 0.0215677593231976 0.000534926551205434 0.00812349855317394 +non-clustered_inv_1-10Kb 1.13644702532971e-19 4.05959323031505e-15 3.55897583131306e-12 0.590290289326669 0.0212169475954757 1.10123020060586e-19 0.177609099239821 0.11282979972687 0.0901241361734239 2.16605622599811e-09 0.0103277672147405 1.73644761214785e-20 1.49696579630446e-20 0.854022364939389 2.53995730561859e-20 1.67991834076044e-20 1.3110361577648e-20 3.51596454806456e-05 2.55596049694926e-20 2.23326137118201e-20 1.87723865406083e-20 0.886957117092274 1.77712011350644e-20 2.20467501238808e-20 1.55467333841473e-20 1.11149468873528e-08 1.90694174609194e-20 0.183125210032775 0.0294871525765127 5.04243710555034e-18 0.0625814611640355 0.00629979985141627 0.00411671328275182 0.0589882409179937 0.0571399826510121 0.0171535945231844 0.0158178600971118 0.000567431853049047 0.114720288814997 0.114146513850172 0.0257651733436543 0.0161703844267467 4.85183154585009e-08 0.0699557017759385 0.053686458847557 0.0407730743937662 0.136827183931607 0.00641968266084143 0.0226793713868238 0.00382709435193121 0.045114109999613 2.20179740219242e-20 3.19927602094556e-20 5.41480440732341e-20 4.1974369108772e-20 3.64885180691663e-20 4.3921818528298e-20 0.820980275443017 3.78655317241334e-20 5.59117980134354e-20 0.182638072848591 0.0525823107785299 0.0380041143320036 0.298468885674946 3.66066275854242e-20 0.00875040706607428 4.72458028745174e-20 3.80676589318223e-20 3.66979116100265e-20 0.0168417063815102 3.39456382921865e-20 0.00709828833206303 0.0143377122779845 0.0792585665443005 0.0347784490419047 0.023591939049409 0.00620015781201765 0.0970620028908303 0.0105770446068158 0.118312524218851 0.011010009267336 0.0183463965501144 0.00406398951126119 0.0518287733764803 0.112164910327406 1.45477090902705e-19 0.129178145482533 8.45408369612758e-20 6.10513091937558e-20 4.19620688953765e-20 0.869385323600842 7.89774967501189e-20 6.84671427066716e-20 9.14098254498316e-20 5.82597787560824e-20 4.80460431933668e-20 3.85987478930723e-20 0.000851879410419281 0.120042862827631 0.00924389248606547 1.7628646352932e-13 0.00690270117378583 2.90817268902941e-20 0.0487389750907749 0.0241446308544105 0.119892488198186 0.00429277136751207 0.0463300444930476 0.0386886954030349 0.0530979524182704 0.0234915217829782 0.021463046484969 0.00237073837232909 2.92061054954554e-09 0.709214691720839 0.00501785139395388 +non-clustered_inv_10-100Kb 0.0125287790626103 8.96489862652724e-20 0.0266829075755633 0.0855343636644458 0.0290430442525159 1.10123020060586e-19 0.0723606129965936 0.0273382389103939 0.038731476614145 0.00311718564449767 1.26653505981664e-11 0.0145570615275005 1.49696579630446e-20 0.104710019325235 0.0568556413885068 1.67991834076044e-20 0.00222202296819453 2.09258714259033e-20 0.027868307254773 0.0229099468532476 0.0157724629560783 0.0117796555507162 0.0291703348733464 2.20467501238808e-20 0.029958097267906 0.00209754232102071 1.82185725733593e-18 0.0720049898049466 0.0386261778319425 3.72788791564996e-05 0.0235263238138651 0.0129689721834315 0.00418794951675156 0.0258182877371746 0.0682491612705167 0.0153351731890443 2.40304114079905e-19 0.0213505378274118 0.0321449469519934 0.0568621564600922 9.38762695860971e-20 0.00548882004407998 0.00372540745489665 0.0348827942475717 0.0322273023260919 0.0378176015621294 0.0351838987778278 2.37822653310514e-05 0.0118725387966254 1.58929435340401e-20 0.0415766276646429 2.20179740219242e-20 0.031487696087882 5.41480440732341e-20 0.0110701132031042 0.018652579045348 0.00365469139533647 0.164844161761801 3.78655317241334e-20 0.0321537038792106 0.0443079192661911 3.1201016757648e-20 0.00975800496470963 0.0588961722501293 8.70958313776144e-13 3.8645359827963e-20 0.00904216878411941 0.00854825242226504 0.0217870665379076 0.042052672785217 3.39456382921865e-20 0.0183010222322308 3.9862459500132e-20 0.031550260106866 0.0154300288093528 7.71429335399417e-20 0.0293341522154845 0.0521260317863484 0.0100822397459057 0.049264831892274 4.25834079286928e-20 0.0528281486022161 0.0397372376479878 0.0218050937382585 3.38022704704076e-07 0.00711758567421917 0.0240957379093579 8.45408369612758e-20 0.0289571508159264 0.0235703753237908 0.0333457796232486 1.58440187552154e-08 6.84671427066716e-20 9.14098254498316e-20 5.82597787560824e-20 0.0137179120000061 0.0248770548629773 3.26120035757344e-20 0.0532063545564295 0.0397982072470862 3.53569749313383e-20 0.00716642533105226 0.00567157683823427 0.0538744624439883 9.91106332588872e-14 0.0236030931218241 0.0358610636140006 0.0087846665282846 0.0390657722492205 0.034931884598668 1.71236872841168e-11 2.94386643348713e-20 0.0328963262851006 0.00207253213750655 0.0104281469256301 0.0334189204027732 +non-clustered_inv_100Kb-1Mb 0.00128840191923552 0.0437086239259989 0.0240796897067372 0.0144382008561452 0.0451187891552324 0.00724286152795328 0.00978502390942759 0.0232259245615718 0.0184695719471576 0.0406240234363839 0.00328076096596802 0.0313340576569002 1.49696579630446e-20 0.00982142687296322 0.0379719536626631 0.00859113929331165 0.00280968015663218 0.0183309501610959 0.00634831801323083 0.0193913066472143 0.0129176315446792 0.00332429635352503 0.00694271557511681 1.6816436358561e-07 0.0462020713376323 1.4436164942841e-20 1.90694174609194e-20 2.00819304717637e-18 0.0110175713201094 1.56314289955482e-19 0.0271948356182907 0.0018630195604947 0.0530116936701445 0.0155948718703358 0.00519965273381916 0.017758821391742 0.0076368882764557 0.0160362533594329 0.0526660788809115 7.55364316665385e-20 9.38762695860971e-20 0.012726453301442 2.04930365936401e-17 0.051952163757043 0.00775432796234208 0.0238031034858985 0.0300362219520974 2.25756462441701e-20 0.00489514377549218 0.00228863299725597 0.062317657231131 0.00100705779569102 0.0188713867914775 0.0298706254777519 0.0276638099160805 0.0475450527485442 0.000671300971109119 4.50470189091067e-20 0.00595011089930893 0.0407072647652672 0.0181966447380811 0.00476581097012984 0.000347366797567647 0.0291768832396592 0.0277574698407634 3.8645359827963e-20 1.18644192376343e-12 0.0189828336417297 0.0214885617807453 0.0383214215611133 3.39456382921865e-20 0.0111714172807601 0.00231268013336195 0.0233883120232859 8.11371711490154e-05 0.0221506812198092 0.0299886680684199 0.0183525257200763 0.0213132274649883 0.0529434618448216 7.92160408538056e-07 0.0260651456868879 0.0856365245978581 9.09254741825586e-20 0.00477865311206688 5.461699605275e-09 0.0413785886005447 8.45408369612758e-20 0.0253900826147704 0.0265791400252815 1.77446343886276e-14 0.0269901420554905 6.16398240113874e-18 0.0133271395902261 0.0135318780032107 1.0163686048525e-17 0.123017150379953 3.26120035757344e-20 0.0429394335896121 0.0257899912907371 3.53569749313383e-20 0.0097951871136939 0.0165512672266548 0.0472998974324979 0.0135064143786057 0.0174177353227607 0.0444681868473917 0.0228050676864632 0.0264902039786593 0.0380867582151659 2.67307083883168e-05 2.26313320271039e-18 0.0452279970520744 0.00798603212862643 0.0174517010661477 0.0272452233701317 +non-clustered_inv_1Mb-10Mb 0.00151556914988262 0.0526747732127574 0.0119147186098626 0.0148713545583637 0.0804304218305075 0.0109154535850036 8.21012451600422e-10 0.0834209243696387 0.0190441902648615 0.0400008805306546 0.0397822312324938 0.188791813505691 7.01987419020952e-10 3.47114419294204e-20 0.0147543269029826 0.0393994076620722 3.56711888511124e-08 0.0274722369546719 2.55596049694926e-20 0.00330919001669443 0.0561615756531104 1.52023047071526e-18 1.77712011350644e-20 0.0118635024553504 0.116045728125146 1.97303082784537e-05 1.43660632115581e-08 0.0170385094773806 0.0195472404507155 0.0243346642135534 0.0403754141507412 0.0237428967294253 0.0223221537505814 0.0764568428591807 0.000384320824490504 0.0060036220508761 0.0198033690168448 7.08490153865265e-20 0.0344151660915733 2.90332170581346e-07 0.0164288622897983 7.36136279388596e-20 0.0221627897740813 0.0474780324960979 0.0209986338224976 1.05461989671689e-19 0.0570015722472446 2.25756462441701e-20 0.0395933283842037 1.58929435340401e-20 0.0734442170219532 2.20179740219242e-20 3.19927602094556e-20 3.09404416411168e-08 0.00586052635028351 0.0773111985604571 4.3921818528298e-20 4.50470189091067e-20 0.0183327705461715 0.00959407461973117 3.07572841045254e-20 3.1201016757648e-20 3.95236128409755e-20 0.0108518145218753 0.198918007946713 2.77194779778442e-08 9.88424169222511e-15 0.0136736175691892 3.66979116100265e-20 0.0642037087895483 3.39456382921865e-20 0.00646904223168011 0.0539442903939039 0.0073692367797711 4.71892355961059e-20 0.0359429901125852 5.64473945464854e-20 0.0124998253240733 0.0259545775654875 0.0734806898969742 0.00932967462261357 7.93159637281987e-09 0.0612147242841325 9.09254741825586e-20 0.0598214516577332 1.45477090902705e-19 0.0363302815789729 0.0450648285434846 0.0933709601866808 0.0226651714018786 0.0242769242288583 7.89774967501189e-20 6.84671427066716e-20 0.0123160189010937 5.82597787560824e-20 0.00402168493636746 0.126286622833213 2.68242964516756e-05 0.210492677836637 4.81650909640983e-20 0.0190025643683047 0.0124293738159308 0.0162417103767523 3.57892404846891e-20 0.036379352371514 0.0582839616532134 0.000162882416123865 0.00839126683942739 0.0114513992046495 0.0235291021372166 0.00319390921934981 0.0222165189036132 0.00755893220610795 0.0247548232402662 4.24615471433568e-20 0.0123684038516136 +non-clustered_inv_>10Mb 0.0354864691272119 0.117781438494498 0.0531560950466159 1.22619698724958e-19 0.00416584346640244 0.0317181285350824 1.8733134051656e-19 0.12425442231183 0.0267344605010766 2.18598846219008e-19 0.0132729164259609 0.205338252869847 1.49696579630446e-20 0.00696596929328923 2.53995730561859e-20 2.01186842831214e-17 0.0326343954980204 0.0463323469200113 2.55596049694926e-20 0.0117180618887113 0.0117267234485349 2.17851047420953e-19 0.02222763111783 0.0344655687484878 0.138414490748604 0.0106059354200373 2.25409780394133e-16 0.0116184319386951 0.037968363756639 1.56314289955482e-19 0.071232755197611 0.057158420872931 0.00577966554345124 0.0772295058884591 0.0086338152592812 0.00124200307527088 0.0477019875347735 0.0110754453901708 8.23014850574395e-20 8.21422300085513e-06 0.0460831823790059 0.00163535545967053 0.0529047531943762 0.134035598050673 1.63426524440033e-19 0.00548748050091715 0.0664250535937905 0.0516176901389254 0.0130145434592528 1.58929435340401e-20 2.2439059470699e-20 0.0162927677911254 0.0136987159239467 0.0164289859155079 4.03179401208821e-07 0.11949294842393 0.0246456494694622 4.50470189091067e-20 0.0161870707119549 5.59117980134354e-20 3.07572841045254e-20 0.012972670530652 3.95236128409755e-20 4.43726306848287e-20 0.213041912947529 0.0499715841085538 4.72458028745174e-20 0.0355929848530512 3.66979116100265e-20 0.0665895709925122 0.0380230461077032 1.97903768206811e-19 0.0251840996120261 0.0219738823990634 0.0382099464818107 0.0489544482951487 0.0157176492990208 0.00330322403869207 0.0117295881685879 0.0356033078153076 0.0372018331962638 0.00934165581483883 0.0700124399112726 0.0442059070563623 0.0772192280036188 1.45477090902705e-19 2.31009450430763e-11 0.0680372649997355 6.10513091937558e-20 0.0878711622905746 9.15434952337402e-05 0.00576142730547209 0.0367784810652497 0.00464750333108726 5.82597787560824e-20 0.00786889190790417 0.155596484004366 3.26120035757344e-20 0.149221383367495 0.0495584782959194 0.0321719083112337 3.33025645875325e-20 0.0299242476279906 0.011268999965188 0.0287433204427323 0.0929995413568686 0.0179298145521549 0.0305216752863265 3.10815414898578e-20 0.00538958851646559 0.0277679017709056 0.00133860509060263 0.0279241163891616 0.0223090057638159 0.00779783865914881 0.0336121388313867 +non-clustered_trans 0.109258721139508 0.105361461850054 0.246019179813602 0.125198242338828 1.28613540869706e-19 1.10123020060586e-19 0.182597615648933 0.235423105698078 0.0670270545314909 0.0786993276394845 0.101292575670976 0.00768699898105436 0.0367093995292618 3.47114419294204e-20 2.53995730561859e-20 3.63677115764154e-14 3.68934674899101e-19 0.807547395458519 2.55596049694926e-20 1.96196830803826e-08 1.87723865406083e-20 0.0240214463614699 0.167185369489791 2.20467501238808e-20 0.377499980428495 0.183938468882797 0.104971842485363 0.0890274677447587 0.135988169114935 0.0561498013383853 0.293990418715899 1.71132737562608e-19 1.54196727801832e-19 0.313623289153621 0.0214538173739653 0.0279812672356162 0.00533588187025713 0.147003545726161 4.10917334915066e-11 0.153591819939 0.186596624958559 2.26759491730983e-06 0.0550696874556812 0.25701091544737 0.0726003253016507 0.0195034085488215 0.386430839660676 0.0384119965831017 0.0131604513924019 0.0267897243217271 0.412823960731438 2.20179740219242e-20 3.19927602094556e-20 5.41480440732341e-20 0.00872609215708369 0.521739891670673 0.202635918739014 0.00349555352465453 3.78655317241334e-20 3.59901463029596e-14 0.656998136667445 0.0231599759547087 0.0411451603588776 4.43726306848287e-20 0.17059871280961 0.0224253270254721 1.95272658994988e-05 0.133580534446142 0.033198256929259 0.372496314206885 0.0799135276769568 0.0497625387727589 0.0470229367424329 0.0286551845490839 0.23271098517923 7.71429335399417e-20 0.056271020020105 0.258050002369932 0.10172080197812 0.140409573418499 0.27510385533021 0.00693775016505639 9.26847697415676e-20 0.000621437372525325 0.557468502519783 0.134360473889807 0.000636549923105609 8.45408369612758e-20 8.52566544114699e-09 0.63075711518505 0.0459522873316469 0.0534913234559102 6.84671427066716e-20 0.177863227711611 5.82597787560824e-20 0.192951058225468 3.85987478930723e-20 0.0293378125205621 1.72779757437759e-19 0.000308985052513299 1.03844487435856e-05 0.047803176478011 0.80205226736425 0.00088803040625432 0.0176645186267774 0.481503957639394 0.0422969362773087 0.178136397726385 3.10815414898578e-20 0.0208715888761797 0.0730522550141182 0.0523737795681177 0.127031467969288 0.159277391743314 0.0861661212911942 0.230970991587199 diff --git a/opencga-storage/opencga-storage-core/src/test/resources/AR2.10039966-01T.copynumber.caveman.vcf.gz b/opencga-storage/opencga-storage-core/src/test/resources/AR2.10039966-01T.copynumber.caveman.vcf.gz new file mode 100644 index 00000000000..3cd9a6f3494 Binary files /dev/null and b/opencga-storage/opencga-storage-core/src/test/resources/AR2.10039966-01T.copynumber.caveman.vcf.gz differ diff --git a/opencga-storage/opencga-storage-core/src/test/resources/AR2.10039966-01T_vs_AR2.10039966-01G.annot.brass.vcf.gz b/opencga-storage/opencga-storage-core/src/test/resources/AR2.10039966-01T_vs_AR2.10039966-01G.annot.brass.vcf.gz new file mode 100644 index 00000000000..10c8532fdd2 Binary files /dev/null and b/opencga-storage/opencga-storage-core/src/test/resources/AR2.10039966-01T_vs_AR2.10039966-01G.annot.brass.vcf.gz differ diff --git a/opencga-storage/opencga-storage-core/src/test/resources/AR2.10039966-01T_vs_AR2.10039966-01G.annot.pindel.vcf.gz b/opencga-storage/opencga-storage-core/src/test/resources/AR2.10039966-01T_vs_AR2.10039966-01G.annot.pindel.vcf.gz new file mode 100644 index 00000000000..69eb2b03002 Binary files /dev/null and b/opencga-storage/opencga-storage-core/src/test/resources/AR2.10039966-01T_vs_AR2.10039966-01G.annot.pindel.vcf.gz differ diff --git a/opencga-storage/opencga-storage-core/src/test/resources/hrdetect_output_38.tsv b/opencga-storage/opencga-storage-core/src/test/resources/hrdetect_output_38.tsv new file mode 100644 index 00000000000..f69d855e19c --- /dev/null +++ b/opencga-storage/opencga-storage-core/src/test/resources/hrdetect_output_38.tsv @@ -0,0 +1,2 @@ +intercept del.mh.prop SNV3 SV3 SV5 hrd SNV8 Probability +AR2.10039966-01T -3.364 6.1388694855816 2.93200322296942 -0.876753168376584 1.62647246717118 -0.102769988932733 0.110259929136027 0.998444003582779 diff --git a/opencga-storage/opencga-storage-core/src/test/resources/mutational-signature-catalogue-snv.json b/opencga-storage/opencga-storage-core/src/test/resources/mutational-signature-catalogue-snv.json new file mode 100644 index 00000000000..1d66d7a23bc --- /dev/null +++ b/opencga-storage/opencga-storage-core/src/test/resources/mutational-signature-catalogue-snv.json @@ -0,0 +1 @@ +{"id":"catalogue-snv-1","description":"Catalogue SNV #1","query":{"sample":"AR2.10039966-01T","fileData":"AR2.10039966-01T_vs_AR2.10039966-01G.annot.muts.caveman.vcf.gz:FILTER=PASS;CLPM=0;ASMD>=140","study":"titi@cancer:steiner","type":"SNV"},"type":"SNV","counts":[{"context":"A[C>A]A","total":244},{"context":"A[C>A]C","total":200},{"context":"A[C>A]G","total":33},{"context":"A[C>A]T","total":215},{"context":"C[C>A]A","total":185},{"context":"C[C>A]C","total":193},{"context":"C[C>A]G","total":30},{"context":"C[C>A]T","total":184},{"context":"G[C>A]A","total":101},{"context":"G[C>A]C","total":123},{"context":"G[C>A]G","total":13},{"context":"G[C>A]T","total":109},{"context":"T[C>A]A","total":204},{"context":"T[C>A]C","total":155},{"context":"T[C>A]G","total":15},{"context":"T[C>A]T","total":203},{"context":"A[C>G]A","total":188},{"context":"A[C>G]C","total":119},{"context":"A[C>G]G","total":42},{"context":"A[C>G]T","total":181},{"context":"C[C>G]A","total":112},{"context":"C[C>G]C","total":105},{"context":"C[C>G]G","total":36},{"context":"C[C>G]T","total":162},{"context":"G[C>G]A","total":94},{"context":"G[C>G]C","total":71},{"context":"G[C>G]G","total":16},{"context":"G[C>G]T","total":125},{"context":"T[C>G]A","total":229},{"context":"T[C>G]C","total":179},{"context":"T[C>G]G","total":27},{"context":"T[C>G]T","total":402},{"context":"A[C>T]A","total":219},{"context":"A[C>T]C","total":109},{"context":"A[C>T]G","total":215},{"context":"A[C>T]T","total":189},{"context":"C[C>T]A","total":119},{"context":"C[C>T]C","total":101},{"context":"C[C>T]G","total":129},{"context":"C[C>T]T","total":182},{"context":"G[C>T]A","total":148},{"context":"G[C>T]C","total":113},{"context":"G[C>T]G","total":161},{"context":"G[C>T]T","total":133},{"context":"T[C>T]A","total":184},{"context":"T[C>T]C","total":159},{"context":"T[C>T]G","total":98},{"context":"T[C>T]T","total":212},{"context":"A[T>A]A","total":93},{"context":"A[T>A]C","total":84},{"context":"A[T>A]G","total":115},{"context":"A[T>A]T","total":176},{"context":"C[T>A]A","total":90},{"context":"C[T>A]C","total":118},{"context":"C[T>A]G","total":139},{"context":"C[T>A]T","total":176},{"context":"G[T>A]A","total":70},{"context":"G[T>A]C","total":61},{"context":"G[T>A]G","total":65},{"context":"G[T>A]T","total":131},{"context":"T[T>A]A","total":131},{"context":"T[T>A]C","total":84},{"context":"T[T>A]G","total":85},{"context":"T[T>A]T","total":182},{"context":"A[T>C]A","total":243},{"context":"A[T>C]C","total":120},{"context":"A[T>C]G","total":174},{"context":"A[T>C]T","total":266},{"context":"C[T>C]A","total":91},{"context":"C[T>C]C","total":122},{"context":"C[T>C]G","total":116},{"context":"C[T>C]T","total":136},{"context":"G[T>C]A","total":123},{"context":"G[T>C]C","total":66},{"context":"G[T>C]G","total":99},{"context":"G[T>C]T","total":136},{"context":"T[T>C]A","total":134},{"context":"T[T>C]C","total":112},{"context":"T[T>C]G","total":73},{"context":"T[T>C]T","total":186},{"context":"A[T>G]A","total":77},{"context":"A[T>G]C","total":32},{"context":"A[T>G]G","total":77},{"context":"A[T>G]T","total":70},{"context":"C[T>G]A","total":45},{"context":"C[T>G]C","total":67},{"context":"C[T>G]G","total":65},{"context":"C[T>G]T","total":108},{"context":"G[T>G]A","total":42},{"context":"G[T>G]C","total":26},{"context":"G[T>G]G","total":69},{"context":"G[T>G]T","total":64},{"context":"T[T>G]A","total":81},{"context":"T[T>G]C","total":46},{"context":"T[T>G]G","total":96},{"context":"T[T>G]T","total":157}],"files":null,"fitting":null,"fittings":null} \ No newline at end of file diff --git a/opencga-storage/opencga-storage-core/src/test/resources/mutational-signature-sv.json b/opencga-storage/opencga-storage-core/src/test/resources/mutational-signature-sv.json new file mode 100644 index 00000000000..170e4753907 --- /dev/null +++ b/opencga-storage/opencga-storage-core/src/test/resources/mutational-signature-sv.json @@ -0,0 +1 @@ +{"id":"catalogue-sv-1","description":"Catalogue SV #1","query":{"sample":"AR2.10039966-01T","type":"DELETION,BREAKEND,DUPLICATION,TANDEM_DUPLICATION,INVERSION,TRANSLOCATION","study":"user@project:study"},"type":"SV","counts":[{"context":"clustered_del_1-10Kb","total":0},{"context":"clustered_del_10-100Kb","total":0},{"context":"clustered_del_100Kb-1Mb","total":0},{"context":"clustered_del_1Mb-10Mb","total":0},{"context":"clustered_del_>10Mb","total":0},{"context":"clustered_tds_1-10Kb","total":0},{"context":"clustered_tds_10-100Kb","total":0},{"context":"clustered_tds_100Kb-1Mb","total":0},{"context":"clustered_tds_1Mb-10Mb","total":0},{"context":"clustered_tds_>10Mb","total":0},{"context":"clustered_inv_1-10Kb","total":0},{"context":"clustered_inv_10-100Kb","total":0},{"context":"clustered_inv_100Kb-1Mb","total":0},{"context":"clustered_inv_1Mb-10Mb","total":0},{"context":"clustered_inv_>10Mb","total":0},{"context":"clustered_trans","total":0},{"context":"non-clustered_del_1-10Kb","total":56},{"context":"non-clustered_del_10-100Kb","total":78},{"context":"non-clustered_del_100Kb-1Mb","total":32},{"context":"non-clustered_del_1Mb-10Mb","total":8},{"context":"non-clustered_del_>10Mb","total":4},{"context":"non-clustered_tds_1-10Kb","total":3},{"context":"non-clustered_tds_10-100Kb","total":6},{"context":"non-clustered_tds_100Kb-1Mb","total":29},{"context":"non-clustered_tds_1Mb-10Mb","total":11},{"context":"non-clustered_tds_>10Mb","total":4},{"context":"non-clustered_inv_1-10Kb","total":7},{"context":"non-clustered_inv_10-100Kb","total":5},{"context":"non-clustered_inv_100Kb-1Mb","total":5},{"context":"non-clustered_inv_1Mb-10Mb","total":5},{"context":"non-clustered_inv_>10Mb","total":3},{"context":"non-clustered_trans","total":24}],"files":null,"fitting":null,"fittings":null} \ No newline at end of file diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml index 26a7790648c..84e193d8263 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/pom.xml @@ -23,7 +23,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java index 9af7402b76a..11d7b22403c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java @@ -73,6 +73,7 @@ import org.opencb.biodata.models.variant.VariantFileMetadata; import org.opencb.biodata.models.variant.avro.VariantType; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.config.storage.StorageEngineConfiguration; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; @@ -423,6 +424,9 @@ static StorageConfiguration updateStorageConfiguration(StorageConfiguration stor options.put(VariantStorageOptions.SPECIES.key(), "hsapiens"); options.put(VariantStorageOptions.ASSEMBLY.key(), "grch37"); +// storageConfiguration.getCellbase().setUrl("https://uk.ws.zettagenomics.com/cellbase/"); +// storageConfiguration.getCellbase().setVersion("v5.2"); +// storageConfiguration.getCellbase().setDataRelease("1"); variantConfiguration.getDatabase().setHosts(Collections.singletonList("hbase://" + HadoopVariantStorageTest.configuration.get().get(HConstants.ZOOKEEPER_QUORUM))); return storageConfiguration; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java index d4fa10432b9..3c6a67074ca 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java @@ -63,6 +63,7 @@ public void before() throws Exception { HadoopVariantStorageEngine variantStorageEngine = getVariantStorageEngine(); variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL); variantStorageEngine.getConfiguration().getCellbase().setVersion("v5.1"); + variantStorageEngine.getConfiguration().getCellbase().setDataRelease("2"); variantStorageEngine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), "grch38"); variantStorageEngine.reloadCellbaseConfiguration(); URI outputUri = newOutputUri(); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml index cee064fdc3b..8ae2892fad5 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-emr6.1/pom.xml @@ -22,7 +22,15 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.4.13-SNAPSHOT +<<<<<<< HEAD +<<<<<<< HEAD + 2.4.11-SNAPSHOT +======= + 2.6.0-SNAPSHOT +>>>>>>> develop +======= + 2.6.0-SNAPSHOT +>>>>>>> develop ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml index c37b4090858..9464e492a22 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp2.6/pom.xml @@ -22,7 +22,15 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.4.13-SNAPSHOT +<<<<<<< HEAD +<<<<<<< HEAD + 2.4.11-SNAPSHOT +======= + 2.6.0-SNAPSHOT +>>>>>>> develop +======= + 2.6.0-SNAPSHOT +>>>>>>> develop ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml index 3cc5434104c..35befe36bc2 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/opencga-storage-hadoop-deps-hdp3.1/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage-hadoop-deps - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml index e52bccb0fb0..98e068c7977 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-deps/pom.xml @@ -50,7 +50,7 @@ org.opencb.opencga opencga-storage-hadoop - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-hadoop/pom.xml b/opencga-storage/opencga-storage-hadoop/pom.xml index 2d7a69a0a7a..32db8a52998 100644 --- a/opencga-storage/opencga-storage-hadoop/pom.xml +++ b/opencga-storage/opencga-storage-hadoop/pom.xml @@ -28,7 +28,7 @@ org.opencb.opencga opencga-storage - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/opencga-storage-mongodb/pom.xml b/opencga-storage/opencga-storage-mongodb/pom.xml index f7304d16df7..016c7f4c509 100644 --- a/opencga-storage/opencga-storage-mongodb/pom.xml +++ b/opencga-storage/opencga-storage-mongodb/pom.xml @@ -22,10 +22,9 @@ org.opencb.opencga opencga-storage - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml - opencga-storage-mongodb jar diff --git a/opencga-storage/opencga-storage-mongodb/src/main/java/org/opencb/opencga/storage/mongodb/variant/adaptors/VariantMongoDBAdaptor.java b/opencga-storage/opencga-storage-mongodb/src/main/java/org/opencb/opencga/storage/mongodb/variant/adaptors/VariantMongoDBAdaptor.java index 31e2ed831be..2b9eb69303e 100644 --- a/opencga-storage/opencga-storage-mongodb/src/main/java/org/opencb/opencga/storage/mongodb/variant/adaptors/VariantMongoDBAdaptor.java +++ b/opencga-storage/opencga-storage-mongodb/src/main/java/org/opencb/opencga/storage/mongodb/variant/adaptors/VariantMongoDBAdaptor.java @@ -17,7 +17,6 @@ package org.opencb.opencga.storage.mongodb.variant.adaptors; import com.mongodb.BasicDBList; -import com.mongodb.MongoClient; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; import htsjdk.variant.vcf.VCFConstants; @@ -313,8 +312,8 @@ private DataResult removeFilesFromVariantsCollection(Bson studiesToRemoveQuery, } Bson update = combine(updates); - logger.debug("removeFile: query = " + query.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); - logger.debug("removeFile: update = " + update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("removeFile: query = " + query.toBsonDocument()); + logger.debug("removeFile: update = " + update.toBsonDocument()); logger.info("Remove files from variants collection - step 2/3"); // Other studies DataResult result2 = getVariantsCollection().update(query, update, new QueryOptions(MULTI, true)); @@ -357,8 +356,8 @@ public DataResult removeStudy(String studyName, long timestamp, QueryOptions opt Bson eq = eq(StageDocumentToVariantConverter.STUDY_FILE_FIELD, studyId.toString()); Bson combine = combine(pull(StageDocumentToVariantConverter.STUDY_FILE_FIELD, studyId.toString()), unset(studyId.toString())); - logger.debug("removeStudy: stage query = " + eq.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); - logger.debug("removeStudy: stage update = " + combine.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("removeStudy: stage query = " + eq.toBsonDocument()); + logger.debug("removeStudy: stage update = " + combine.toBsonDocument()); logger.info("Remove study from stage collection - step 1/" + (purge ? '2' : '1')); getStageCollection(studyId).update(eq, combine, new QueryOptions(MULTI, true)); @@ -376,8 +375,8 @@ private DataResult removeStudyFromVariants(int studyId, Bson query, long timesta pull(DocumentToVariantConverter.STATS_FIELD, eq(DocumentToVariantStatsConverter.STUDY_ID, studyId)), getSetIndexNotSynchronized(timestamp) ); - logger.debug("removeStudy: query = {}", query.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); - logger.debug("removeStudy: update = {}", update.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry())); + logger.debug("removeStudy: query = {}", query.toBsonDocument()); + logger.debug("removeStudy: update = {}", update.toBsonDocument()); DataResult result = variantsCollection.update(query, update, new QueryOptions(MULTI, true)); logger.debug("removeStudy: matched = {}", result.getNumMatches()); @@ -493,7 +492,8 @@ public VariantQueryResult get(ParsedVariantQuery variantQuery, QueryOpt if (options.getBoolean("explain", false)) { Document explain = variantsCollection.nativeQuery().explain(mongoQuery, projection, options); - logger.debug("MongoDB Explain = {}", explain.toJson(new JsonWriterSettings(JsonMode.SHELL, true))); + logger.debug("MongoDB Explain = {}", + explain.toJson(JsonWriterSettings.builder().outputMode(JsonMode.SHELL).indent(true).build())); } DocumentToVariantConverter converter = getDocumentToVariantConverter(variantQuery.getQuery(), variantQueryProjection); diff --git a/opencga-storage/opencga-storage-mongodb/src/main/java/org/opencb/opencga/storage/mongodb/variant/adaptors/VariantMongoDBQueryParser.java b/opencga-storage/opencga-storage-mongodb/src/main/java/org/opencb/opencga/storage/mongodb/variant/adaptors/VariantMongoDBQueryParser.java index 9c284ac441e..a9e9eae5e42 100644 --- a/opencga-storage/opencga-storage-mongodb/src/main/java/org/opencb/opencga/storage/mongodb/variant/adaptors/VariantMongoDBQueryParser.java +++ b/opencga-storage/opencga-storage-mongodb/src/main/java/org/opencb/opencga/storage/mongodb/variant/adaptors/VariantMongoDBQueryParser.java @@ -20,11 +20,14 @@ import com.mongodb.BasicDBObject; import com.mongodb.DBObject; import com.mongodb.QueryBuilder; +import com.mongodb.client.model.Accumulators; +import com.mongodb.client.model.Filters; import htsjdk.variant.vcf.VCFConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.bson.Document; +import org.bson.conversions.Bson; import org.bson.json.JsonMode; import org.bson.json.JsonWriterSettings; import org.opencb.biodata.models.core.Region; @@ -85,7 +88,8 @@ protected Document parseQuery(Query query) { } protected Document parseQuery(ParsedVariantQuery parsedVariantQuery) { - QueryBuilder builder = new QueryBuilder(); + + List bsonList = new ArrayList<>(); if (parsedVariantQuery != null) { // Copy given query. It may be modified Query query = new Query(parsedVariantQuery.getInputQuery()); @@ -96,7 +100,7 @@ protected Document parseQuery(ParsedVariantQuery parsedVariantQuery) { nonGeneRegionFilter = true; List regions = Region.parseRegions(query.getString(REGION.key()), true); if (!regions.isEmpty()) { - getRegionFilter(regions, builder); + getRegionFilter(regions, bsonList); } } @@ -107,8 +111,8 @@ protected Document parseQuery(ParsedVariantQuery parsedVariantQuery) { addQueryStringFilter(DocumentToVariantConverter.ANNOTATION_FIELD + '.' + DocumentToVariantAnnotationConverter.XREFS_FIELD + '.' + DocumentToVariantAnnotationConverter.XREF_ID_FIELD, - variantQueryXref.getIds(), builder, QueryOperation.OR); - addQueryStringFilter(DocumentToVariantConverter.IDS_FIELD, variantQueryXref.getIds(), builder, QueryOperation.OR); + variantQueryXref.getIds(), bsonList, QueryOperation.OR); + addQueryStringFilter(DocumentToVariantConverter.IDS_FIELD, variantQueryXref.getIds(), bsonList, QueryOperation.OR); } if (!variantQueryXref.getOtherXrefs().isEmpty()) { @@ -116,7 +120,7 @@ protected Document parseQuery(ParsedVariantQuery parsedVariantQuery) { addQueryStringFilter(DocumentToVariantConverter.ANNOTATION_FIELD + '.' + DocumentToVariantAnnotationConverter.XREFS_FIELD + '.' + DocumentToVariantAnnotationConverter.XREF_ID_FIELD, - variantQueryXref.getOtherXrefs(), builder, QueryOperation.OR); + variantQueryXref.getOtherXrefs(), bsonList, QueryOperation.OR); } List idIntersect = query.getAsStringList(ID_INTERSECT.key()).stream().map(Variant::new).collect(Collectors.toList()); diff --git a/opencga-storage/opencga-storage-server/pom.xml b/opencga-storage/opencga-storage-server/pom.xml index 73afe69987d..59eafb048cc 100644 --- a/opencga-storage/opencga-storage-server/pom.xml +++ b/opencga-storage/opencga-storage-server/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga-storage - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/opencga-storage/pom.xml b/opencga-storage/pom.xml index edad9bd8db8..73ccc6fbdd0 100644 --- a/opencga-storage/pom.xml +++ b/opencga-storage/pom.xml @@ -22,7 +22,7 @@ org.opencb.opencga opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml @@ -37,7 +37,6 @@ opencga-storage-app opencga-storage-core - opencga-storage-mongodb opencga-storage-hadoop opencga-storage-server opencga-storage-benchmark diff --git a/opencga-test/pom.xml b/opencga-test/pom.xml index b11bf7e5c5c..a6a895e02c7 100644 --- a/opencga-test/pom.xml +++ b/opencga-test/pom.xml @@ -24,7 +24,7 @@ org.opencb.opencga opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index d282415e6e2..59d69b15819 100644 --- a/pom.xml +++ b/pom.xml @@ -22,9 +22,8 @@ org.opencb.opencga opencga - 2.4.13-SNAPSHOT + 2.6.0-SNAPSHOT pom - OpenCGA OenCGA projects implements a big data platform for genomic data analysis http://docs.opencb.org/display/opencga/ @@ -43,11 +42,11 @@ - 2.4.12_dev - 2.4.12_dev - 5.1.6 - 2.4.10-SNAPSHOT - 4.4.5-SNAPSHOT + 2.6.0_dev + 2.6.0_dev + 5.3.0-SNAPSHOT + 2.6.0-SNAPSHOT + 4.6.0-SNAPSHOT 2.4.10 0.2.0 diff --git a/script.sh b/script.sh deleted file mode 100644 index a9bf588e2f8..00000000000 --- a/script.sh +++ /dev/null @@ -1 +0,0 @@ -#!/bin/bash diff --git a/test_script.sh b/test_script.sh deleted file mode 100755 index fe344d69edc..00000000000 --- a/test_script.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -./build/bin/opencga.sh --shell -samples search - - - -