TASK-6757 - Deployment: Add Jupyter Notebooks feature

.github/workflows/manual-deploy-docker.yml

@@ -45,7 +45,7 @@ jobs:
             ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           fi
       - name: Maven Build (skip tests)
-        run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }} -Dopencga.war.name=opencga -Dcheckstyle.skip
+        run: mvn -T 2 clean install -DskipTests -P${{ inputs.hadoop }} -Dopencga.war.name=opencga -Dcheckstyle.skip --no-transfer-progress
       - uses: actions/upload-artifact@v4
         with:
           name: build-folder

.github/workflows/manual-deploy-ext-tools.yml

@@ -40,7 +40,7 @@ jobs:
             ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
           fi
       - name: Maven Build (skip tests)
-        run: mvn -T 2 clean install -DskipTests
+        run: mvn -T 2 clean install -DskipTests --no-transfer-progress
       - uses: actions/upload-artifact@v4
         with:
           name: build-folder

.github/workflows/manual-deploy-python-notebook.yml

@@ -0,0 +1,60 @@
+name: Reusable deploy Docker python-notebook
+run-name: 'Run deploy Docker python-notebook ${{ inputs.branch }} by @${{ github.actor }}'
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "The branch, tag or SHA of the source code to build docker."
+        type: string
+        required: true
+      tag:
+        description: "The tag for the new docker."
+        type: string
+        required: true
+      hadoop:
+        type: string
+        description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]'
+        required: false
+        default: "hdp3.1"
+      pyopencga_version:
+        type: string
+        description: 'PyOpenCGA version.'
+        required: true
+
+jobs:
+  build:
+    name: Build Java app
+    runs-on: ubuntu-22.04
+    outputs:
+      version: ${{ steps.get_project_version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+          ref: "${{ inputs.branch }}"
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
+      - name: Install dependencies branches
+        run: |
+          if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
+            chmod +x ./.github/workflows/scripts/get_same_branch.sh
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
+          fi
+      - name: Maven Build (skip tests)
+        run: mvn -T 2 clean install -DskipTests -Dmaven.test.skip=true --no-transfer-progress
+      - uses: actions/upload-artifact@v4
+        with:
+          name: build-folder
+          path: build
+
+  deploy-docker-python-notebook:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: build
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images python-notebook --tag ${{ inputs.tag }} --docker-build-args "--build-arg VERSION=${{ inputs.pyopencga_version }}"
+    secrets: inherit
+

.github/workflows/release.yml

@@ -39,10 +39,17 @@ jobs:
       artifact: build-folder
     secrets: inherit
 
+  deploy-docker-python-notebook:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: [build-hdp, deploy-python]
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images python-notebook --tag ${{ needs.build-hdp.outputs.version }} --docker-build-args "--build-arg VERSION=${{ needs.build-hdp.outputs.version }}"
+    secrets: inherit
+
   # Add the release job that depends on all the previous jobs
   release:
     uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@develop
-    needs: [ build-hdp, deploy-docker-hdp, deploy-maven, deploy-python]
+    needs: [ build-hdp, deploy-docker-hdp, deploy-maven, deploy-python, deploy-docker-python-notebook]
     with:
       artifact: build-folder
       file: |

opencga-analysis/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>3.3.0</version>
+        <version>3.4.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java

@@ -1757,8 +1757,7 @@ public static DataStore defaultDataStore(CatalogManager catalogManager, Project
         return defaultDataStore(catalogManager.getConfiguration().getDatabasePrefix(), project.getFqn());
     }
 
-    public static DataStore defaultDataStore(String databasePrefix, String projectFqnStr)
-            throws CatalogException {
+    public static DataStore defaultDataStore(String databasePrefix, String projectFqnStr) {
         CatalogFqn projectFqn = CatalogFqn.extractFqnFromProjectFqn(projectFqnStr);
 
         String dbName = buildDatabaseName(databasePrefix, projectFqn.getOrganizationId(), projectFqn.getProjectId());

opencga-app/app/cloud/docker/docker-build.py

@@ -11,7 +11,7 @@
 parser = argparse.ArgumentParser()
 parser.add_argument('action', help="Action to execute", choices=["build", "push", "delete"], default="build")
 parser.add_argument('--images',
-                    help="comma separated list of images to be made, e.g. base,init,ext-tools,r-builder")
+                    help="comma separated list of images to be made, e.g. base,init,ext-tools,python-notebook,r-builder")
 parser.add_argument('--tag', help="the tag for this code, e.g. v2.0.0-hdp3.1")
 parser.add_argument('--build-folder', help="the location of the build folder, if not default location")
 parser.add_argument('--org', help="Docker organization", default="opencb")
@@ -201,7 +201,7 @@ def delete():
 
 # 5. Get a list with all images
 if args.images is None:
-    images = ["base", "init", "demo", "r", "samtools", "ext-tools"]
+    images = ["base", "init", "demo", "r", "samtools", "python-notebook", "ext-tools"]
 else:
     imagesUnsorted = args.images.split(",")
     images = []

opencga-app/app/cloud/docker/opencga-python-notebook/Dockerfile

@@ -0,0 +1,13 @@
+# Start from the Jupyter scipy-notebook image
+FROM quay.io/jupyter/scipy-notebook:ubuntu-24.04
+ARG VERSION
+USER root
+
+RUN apt-get update && apt-get -y upgrade && \
+    ## Install pyopencga ${VERSION}
+    pip install pyopencga==${VERSION} && \
+    ## Clean downloaded packages to make images smaller
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+## Run Docker images as non root
+USER ${NB_UID}

opencga-app/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>3.3.0</version>
+        <version>3.4.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/MigrationCommandExecutor.java

@@ -70,7 +70,7 @@ private void runMigrationToV3() throws Exception {
         setCatalogDatabaseCredentials(options, options.commonOptions);
 
         OrganizationMigration organizationMigration = new OrganizationMigration(configuration, options.commonOptions.adminPassword,
-                options.user);
+                options.user, options.organizationId, Paths.get(appHome));
         organizationMigration.execute();
     }
 

opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/options/MigrationCommandOptions.java

@@ -46,6 +46,10 @@ public class OrganizationMigrationCommandOptions extends AdminCliOptionsParser.C
         @Parameter(names = {"--user"}, description = "User whose data is going to be migrated. If more than one user of type FULL contains"
                 + " projects and studies, only the one provided will keep the data and will be fully migrated.")
         public String user;
+
+        @Parameter(names = {"--organization-id"}, description = "Optional parameter to specify how the new organization will be named." +
+                " By default, if not provided, the organization id will match the user id that is currently owning the data.")
+        public String organizationId;
     }
 
     @Parameters(commandNames = {"summary"}, commandDescription = "Obtain migrations status summary")

opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_0_0/OrganizationMigration.java

@@ -4,12 +4,15 @@
 import org.opencb.opencga.catalog.migration.MigrationTool;
 import org.opencb.opencga.core.config.Configuration;
 
+import java.nio.file.Path;
+
 @Migration(id = "add_organizations", description = "Add new Organization layer #TASK-4389", version = "3.0.0",
-        language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20231212,
+        language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20231212, manual = true,
         deprecatedSince = "3.1.0")
+
 public class OrganizationMigration extends MigrationTool {
 
-    public OrganizationMigration(Configuration configuration, String adminPassword, String userId) {
+    public OrganizationMigration(Configuration configuration, String adminPassword, String userId, String organizationId, Path appHome) {
     }
 
     @Override

opencga-catalog/pom.xml

@@ -23,7 +23,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>3.3.0</version>
+        <version>3.4.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java

@@ -239,8 +239,8 @@ public void checkCanAssignOrSeePermissions(String organizationId, long studyId,
 
     @Override
     public void checkCanCreateUpdateDeleteVariableSets(String organizationId, long studyId, String userId) throws CatalogException {
-        if (!isAtLeastOrganizationOwnerOrAdmin(organizationId, userId)) {
-            throw CatalogAuthorizationException.notOrganizationOwnerOrAdmin("create, update or delete variable sets.");
+        if (!isAtLeastStudyAdministrator(organizationId, studyId, userId)) {
+            throw CatalogAuthorizationException.notStudyAdmin("create, update or delete variable sets.");
         }
     }
 

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AnnotationMongoDBAdaptor.java

@@ -70,11 +70,23 @@ public abstract class AnnotationMongoDBAdaptor<T> extends CatalogMongoDBAdaptor
 
     protected abstract MongoDBCollection getCollection();
 
-    abstract OpenCGAResult<T> transactionalUpdate(ClientSession clientSession, T entry, ObjectMap parameters,
+    OpenCGAResult<T> transactionalUpdate(ClientSession clientSession, T entry, ObjectMap parameters,
                                                   List<VariableSet> variableSetList, QueryOptions queryOptions)
+            throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException {
+        return transactionalUpdate(clientSession, entry, parameters, variableSetList, queryOptions, true);
+    }
+
+    abstract OpenCGAResult<T> transactionalUpdate(ClientSession clientSession, T entry, ObjectMap parameters,
+                                                  List<VariableSet> variableSetList, QueryOptions queryOptions, boolean incrementVersion)
             throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException;
 
-    abstract OpenCGAResult<T> transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, UpdateDocument updateDocument)
+    OpenCGAResult<T> transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, UpdateDocument updateDocument)
+            throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException {
+        return transactionalUpdate(clientSession, studyUid, query, updateDocument, true);
+    }
+
+    abstract OpenCGAResult<T> transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, UpdateDocument updateDocument,
+                                         boolean incrementVersion)
             throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException;
 
     public enum AnnotationSetParams implements QueryParam {

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java

@@ -355,20 +355,21 @@ OpenCGAResult transactionalUpdate(ClientSession clientSession, ClinicalAnalysis
 
     @Override
     OpenCGAResult<ClinicalAnalysis> transactionalUpdate(ClientSession clientSession, ClinicalAnalysis entry, ObjectMap parameters,
-                                                        List<VariableSet> variableSetList, QueryOptions queryOptions)
+                                                        List<VariableSet> variableSetList, QueryOptions queryOptions,
+                                                        boolean incrementVersion)
             throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException {
         throw new NotImplementedException("Please call to the other transactionalUpdate method passing the ClinicalAudit list");
     }
 
     @Override
     OpenCGAResult<ClinicalAnalysis> transactionalUpdate(ClientSession clientSession, long studyUid, Bson query,
-                                                        UpdateDocument updateDocument)
+                                                        UpdateDocument updateDocument, boolean incrementVersion)
             throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException {
         long tmpStartTime = startQuery();
 
         Document updateOperation = updateDocument.toFinalUpdateDocument();
         if (!updateOperation.isEmpty()) {
-            return versionedMongoDBAdaptor.update(clientSession, query, entryList -> {
+            SnapshotVersionedMongoDBAdaptor.FunctionWithException<ClinicalAnalysis> updateClinicalReferences = (clinicalList) -> {
                 logger.debug("Update clinical analysis. Query: {}, Update: {}", query.toBsonDocument(), updateDocument);
                 DataResult<?> update = clinicalCollection.update(clientSession, query, updateOperation, null);
 
@@ -385,7 +386,12 @@ OpenCGAResult<ClinicalAnalysis> transactionalUpdate(ClientSession clientSession,
 
                 logger.debug("{} clinical analyses successfully updated", update.getNumUpdated());
                 return endWrite(tmpStartTime, update.getNumMatches(), update.getNumUpdated(), Collections.emptyList());
-            }, null, null);
+            };
+            if (incrementVersion) {
+                return versionedMongoDBAdaptor.update(clientSession, query, null, updateClinicalReferences, null, null);
+            } else {
+                return versionedMongoDBAdaptor.updateWithoutVersionIncrement(clientSession, query, null, updateClinicalReferences);
+            }
         } else {
             throw new CatalogDBException("Nothing to update");
         }

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/CohortMongoDBAdaptor.java

@@ -286,7 +286,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, List<VariableSet>
 
     @Override
     OpenCGAResult<Cohort> transactionalUpdate(ClientSession clientSession, Cohort cohort, ObjectMap parameters,
-                                              List<VariableSet> variableSetList, QueryOptions queryOptions)
+                                              List<VariableSet> variableSetList, QueryOptions queryOptions, boolean incrementVersion)
             throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException {
         long tmpStartTime = startQuery();
         Query tmpQuery = new Query()
@@ -341,7 +341,8 @@ OpenCGAResult<Cohort> transactionalUpdate(ClientSession clientSession, Cohort co
     }
 
     @Override
-    OpenCGAResult<Cohort> transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, UpdateDocument updateDocument)
+    OpenCGAResult<Cohort> transactionalUpdate(ClientSession clientSession, long studyUid, Bson query, UpdateDocument updateDocument,
+                                              boolean incrementVersion)
             throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException {
         long tmpStartTime = startQuery();