Merge branch 'develop' of github.com:opencb/opencga into develop

opencb · Dec 18, 2024 · 6336ac1 · 6336ac1
2 parents 33d55c9 + 98c9b19
commit 6336ac1
Show file tree

Hide file tree

Showing 51 changed files with 912 additions and 273 deletions.
diff --git a/README.md b/README.md
@@ -6,19 +6,13 @@ OpenCGA constitutes the big data analysis component of [OpenCB](http://www.openc
 ### Documentation
 You can find OpenCGA documentation and tutorials at: https://github.com/opencb/opencga/wiki.
 
-### Issues Tracking
-You can report bugs or request new features at [GitHub issue tracking](https://github.com/opencb/opencga/issues).
+### Feedback and Feature Requests
+Found a bug or have an idea for a new feature? Let us know at zettagenomics.com/academic.
 
 ### Release Notes and Roadmap
-Releases notes are available at [GitHub releases](https://github.com/opencb/opencga/releases).
-
-Roadmap is available at [GitHub milestones](https://github.com/opencb/opencga/milestones). You can report bugs or request new features at [GitHub issue tracking](https://github.com/opencb/opencga/issues).
-
-### Versioning
-OpenCGA is versioned following the rules from [Semantic versioning](https://semver.org/).
+Releases notes are available at https://zettagenomics.com/release-notes/
 
 ### Maintainers
-We recommend to contact OpenCGA developers by writing to OpenCB mailing list [email protected]. Current main developers and maintainers are:
 * Ignacio Medina ([email protected]) (_Founder and Project Leader_)
 * Jacobo Coll ([email protected])
 * Pedro Furio ([email protected])
@@ -33,7 +27,7 @@ We recommend to contact OpenCGA developers by writing to OpenCB mailing list ope
 * Franscisco Salavert ([email protected])
 
 ##### Contributing
-OpenCGA is an open-source and collaborative project. We appreciate any help and feedback from users, you can contribute in many different ways such as simple bug reporting and feature request. Dependending on your skills you are more than welcome to develop client tools, new features or even fixing bugs.
+OpenCGA is an open-source and collaborative project. We appreciate any help and feedback from users, you can contribute in many different ways such as simple bug reporting and feature request. Dependending on your skills you are more than welcome to develop client tools, new features, or even fix bugs.
 
 
 # How to build

diff --git a/...alysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java b/...alysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java
@@ -23,7 +23,9 @@
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
 import org.opencb.opencga.core.models.common.Enums;
+import org.opencb.opencga.core.models.common.InternalStatus;
 import org.opencb.opencga.core.models.file.File;
+import org.opencb.opencga.core.models.file.FileInternalCoverageIndex;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;
 
@@ -40,7 +42,7 @@
 public class AlignmentCoverageAnalysis extends OpenCgaToolScopeStudy {
 
     public static final String ID = "coverage-index-run";
-    public static final String DESCRIPTION = "Compute the coverage from a given alignment file, e.g., create a "
+    public static final String DESCRIPTION = "Compute the coverage from a given BAM alignment file, e.g., create a "
             + AlignmentConstants.BIGWIG_EXTENSION + " file from a " + AlignmentConstants.BAM_EXTENSION + " file";
 
     @ToolParams
@@ -64,37 +66,36 @@ protected void check() throws Exception {
 
         //  Checking BAM file ID
         try {
-            bamCatalogFile = catalogManager.getFileManager().get(getStudy(), coverageParams.getBamFileId(), QueryOptions.empty(),
+            bamCatalogFile = catalogManager.getFileManager().get(getStudy(), coverageParams.getFileId(), QueryOptions.empty(),
                     getToken()).first();
             if (bamCatalogFile == null) {
-                throw new ToolException("Could not find BAM file from ID '" + coverageParams.getBamFileId() + "'");
+                throw new ToolException("Could not find BAM file from ID '" + coverageParams.getFileId() + "'");
             }
         } catch (Exception e) {
-            throw new ToolException("Could not get BAM file from ID " + coverageParams.getBamFileId());
+            throw new ToolException("Could not get BAM file from ID " + coverageParams.getFileId());
         }
 
         // Check if the input file is .bam
         if (!bamCatalogFile.getName().endsWith(AlignmentConstants.BAM_EXTENSION)) {
-            throw new ToolException("Invalid input alignment file '" + coverageParams.getBamFileId() + "' (" + bamCatalogFile.getName()
+            throw new ToolException("Invalid input alignment file '" + coverageParams.getFileId() + "' (" + bamCatalogFile.getName()
                     + "): it must be in BAM format");
         }
 
         // Getting BAI file
-        String baiFileId = coverageParams.getBaiFileId();
+        String baiFileId;
+        try {
+            baiFileId = bamCatalogFile.getInternal().getAlignment().getIndex().getFileId();
+        } catch (Exception e) {
+            throw new ToolException("Could not get internal alignment index file Id from BAM file ID '" + bamCatalogFile.getId() + "'");
+        }
         if (StringUtils.isEmpty(baiFileId)) {
-            // BAI file ID was not provided, looking for it
-            logger.info("BAI file ID was not provided, getting it from the internal alignment index of the BAM file ID {}",
-                    bamCatalogFile.getId());
-            try {
-                baiFileId = bamCatalogFile.getInternal().getAlignment().getIndex().getFileId();
-            } catch (Exception e) {
-                throw new ToolException("Could not get internal alignment index file Id from BAM file ID '" + bamCatalogFile.getId());
-            }
+            throw new ToolException("Could not find the alignment index file for the BAM file ID '" + bamCatalogFile.getId() + "'. Please,"
+                    + " create the alignment index file before computing the coverage");
         }
         try {
             baiCatalogFile = catalogManager.getFileManager().get(getStudy(), baiFileId, QueryOptions.empty(), getToken()).first();
             if (baiCatalogFile == null) {
-                throw new ToolException("Could not find BAI file from ID '" + coverageParams.getBaiFileId() + "'");
+                throw new ToolException("Could not find BAI file from ID '" + baiFileId + "'");
             }
         } catch (Exception e) {
             throw new ToolException("Could not get BAI file from file ID " + baiFileId);
@@ -115,6 +116,18 @@ protected void check() throws Exception {
             coverageParams.setWindowSize(Integer.parseInt(COVERAGE_WINDOW_SIZE_DEFAULT));
             logger.info("Window size is set to {}", coverageParams.getWindowSize());
         }
+
+        // Check overwrite
+        String bwFileId;
+        try {
+            bwFileId = bamCatalogFile.getInternal().getAlignment().getCoverage().getFileId();
+        } catch (Exception e) {
+            bwFileId = null;
+        }
+        if (StringUtils.isNotEmpty(bwFileId) && !coverageParams.isOverwrite()) {
+            throw new ToolException("Coverage file ID '" + bwFileId + "' already exists for file ID '" + bamCatalogFile.getId()
+                    + "'. To overwrite the BIGWIG file use the flag --overwrite");
+        }
     }
 
     @Override
@@ -192,7 +205,12 @@ protected void run() throws Exception {
             }
 
             // Link generated BIGWIG file and update samples info
-            AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, getJobId(), study, catalogManager, token);
+            File bwCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, getJobId(), study, catalogManager, token);
+
+            // Update BAM file internal in order to set the coverage index (BIGWIG)
+            FileInternalCoverageIndex fileCoverageIndex = new FileInternalCoverageIndex(new InternalStatus(InternalStatus.READY),
+                    bwCatalogFile.getId(), "deeptools bamCoverage", coverageParams.getWindowSize());
+            catalogManager.getFileManager().updateFileInternalCoverageIndex(study, bamCatalogFile, fileCoverageIndex, token);
         });
     }
 }
diff --git a/...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java b/...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentIndexOperation.java
@@ -19,15 +19,14 @@
 import org.apache.commons.lang3.StringUtils;
 import org.opencb.biodata.tools.alignment.BamManager;
 import org.opencb.commons.datastore.core.QueryOptions;
-import org.opencb.opencga.analysis.tools.OpenCgaTool;
 import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.alignment.AlignmentIndexParams;
-import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.common.InternalStatus;
-import org.opencb.opencga.core.models.file.*;
+import org.opencb.opencga.core.models.file.File;
+import org.opencb.opencga.core.models.file.FileInternalAlignmentIndex;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;
@@ -40,7 +39,12 @@
 public class AlignmentIndexOperation extends OpenCgaToolScopeStudy {
 
     public static final String ID = "alignment-index-run";
-    public static final String DESCRIPTION = "Index a given alignment file, e.g., create a .bai file from a .bam file";
+    public static final String DESCRIPTION = "Index a given alignment file BAM/CRAM, e.g., create a " + AlignmentConstants.BAI_EXTENSION
+            + " file from a " + AlignmentConstants.BAM_EXTENSION + " file";
+
+    private String study;
+    private String inputFile;
+    private boolean overwrite = false;
 
     @ToolParams
     protected final AlignmentIndexParams indexParams = new AlignmentIndexParams();
@@ -81,6 +85,18 @@ protected void check() throws Exception {
             throw new ToolException("Invalid input alignment file '" + indexParams.getFileId() + "': it must be in BAM or CRAM format");
         }
 
+        // Check overwrite
+        String baiFileId;
+        try {
+            baiFileId = inputCatalogFile.getInternal().getAlignment().getIndex().getFileId();
+        } catch (Exception e) {
+            baiFileId = null;
+        }
+        if (StringUtils.isNotEmpty(baiFileId) && !overwrite) {
+            throw new ToolException("Alignment index file ID '" + baiFileId + "' already exists for file ID '" + inputCatalogFile.getId()
+                    + "'. To overwrite the alignment index file use the flag --overwrite");
+        }
+
         outputPath = getOutDir().resolve(filename + (filename.endsWith(AlignmentConstants.BAM_EXTENSION)
                 ? AlignmentConstants.BAI_EXTENSION : AlignmentConstants.CRAI_EXTENSION));
     }
@@ -131,4 +147,31 @@ protected void run() throws Exception {
             catalogManager.getFileManager().updateFileInternalAlignmentIndex(study, inputCatalogFile, fileAlignmentIndex, token);
         });
     }
+
+    public String getStudy() {
+        return study;
+    }
+
+    public AlignmentIndexOperation setStudy(String study) {
+        this.study = study;
+        return this;
+    }
+
+    public String getInputFile() {
+        return inputFile;
+    }
+
+    public AlignmentIndexOperation setInputFile(String inputFile) {
+        this.inputFile = inputFile;
+        return this;
+    }
+
+    public boolean isOverwrite() {
+        return overwrite;
+    }
+
+    public AlignmentIndexOperation setOverwrite(boolean overwrite) {
+        this.overwrite = overwrite;
+        return this;
+    }
 }
diff --git a/...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java b/...analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentStorageManager.java
@@ -88,19 +88,6 @@ public AlignmentStorageManager(CatalogManager catalogManager, StorageEngineFacto
         initStatsMap();
     }
 
-//    //-------------------------------------------------------------------------
-//    // INDEX
-//    //-------------------------------------------------------------------------
-//
-//    public void index(String study, String inputFile, String outdir, String token) throws ToolException {
-//        ToolRunner toolRunner = new ToolRunner("", catalogManager, storageEngineFactory);
-//
-//        AlignmentIndexParams params = new AlignmentIndexParams();
-//        params.setFileId(inputFile);
-//        toolRunner.execute(AlignmentIndexOperation.class, params, new ObjectMap(ParamConstants.STUDY_PARAM, study), Paths.get(outdir),
-//                jobId, token);
-//    }
-
     //-------------------------------------------------------------------------
     // QUERY
     //-------------------------------------------------------------------------

diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/AssociateAlignmentFiles.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/AssociateAlignmentFiles.java
@@ -0,0 +1,18 @@
+package org.opencb.opencga.analysis.file;
+
+import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
+import org.opencb.opencga.core.models.common.Enums;
+import org.opencb.opencga.core.tools.annotations.Tool;
+
+@Tool(id = AssociateAlignmentFiles.ID, resource = Enums.Resource.FILE, type = Tool.Type.OPERATION,
+        description = "Automatically associate alignment files with its index and coverage files.", priority = Enums.Priority.LOW)
+public class AssociateAlignmentFiles extends OpenCgaToolScopeStudy {
+
+    public final static String ID = "associate-alignment-files";
+
+    @Override
+    protected void run() throws Exception {
+        catalogManager.getFileManager().associateAlignmentFiles(getStudyFqn(), token);
+    }
+
+}
diff --git a/...a-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java b/...a-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java
@@ -371,8 +371,7 @@ public void testNonReadOnlyCoverageIndex() throws Exception {
 
         // Run coverage index
         CoverageIndexParams coverageOarams = new CoverageIndexParams();
-        coverageOarams.setBamFileId(bamFile.getId());
-        coverageOarams.setBaiFileId(baiFile.getId());
+        coverageOarams.setFileId(bamFile.getId());
         Path coverageIndexOutdir = Paths.get(opencga.createTmpOutdir("_coverage_index"));
         toolRunner.execute(AlignmentCoverageAnalysis.class, coverageOarams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), coverageIndexOutdir, "jobId-readonly-coverage-index", false, token);
 
@@ -416,8 +415,7 @@ public void testReadOnlyCoverageIndex() throws Exception {
 
         // Run coverage index
         CoverageIndexParams coverageOarams = new CoverageIndexParams();
-        coverageOarams.setBamFileId(bamFile.getId());
-        coverageOarams.setBaiFileId(baiFile.getId());
+        coverageOarams.setFileId(bamFile.getId());
         Path coverageIndexOutdir = Paths.get(opencga.createTmpOutdir("_coverage_index"));
         toolRunner.execute(AlignmentCoverageAnalysis.class, coverageOarams, new ObjectMap(ParamConstants.STUDY_PARAM, STUDY), coverageIndexOutdir, "jobId-readonly-coverage-index", false, token);