Merge branch 'master' into snyk-fix-2c991c74345c1d96c136dc86c1031066

uc-cdis · Oct 30, 2024 · 8256d07 · 8256d07
2 parents 2bb0cc5 + be15542
commit 8256d07
Show file tree

Hide file tree

Showing 125 changed files with 10,317 additions and 24,427 deletions.
diff --git a/.github/workflows/build_and_push_nf_base_images.yml b/.github/workflows/build_and_push_nf_base_images.yml
@@ -0,0 +1,85 @@
+name: Push new nextflow images to ECR
+on:
+  workflow_dispatch:
+  push:
+    branches: master
+    paths:
+      - 'nextflow-base-images/**' #Runs every time this folder gets updated
+      - '.github/workflows/build_and_push_nf_base_images.yml'
+  schedule:
+    - cron: '0 0 * * 6'  # Run at midnight UTC every Saturday
+
+jobs:
+  build_and_push_nf_images:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          path: containers
+
+
+      - name: Checkout other private repository
+        uses: actions/checkout@v3
+        with:
+          repository: uc-cdis/base-images
+          token: ${{ secrets.PLANXCYBORG_TOKEN }}
+          path: base-images
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Set up AWS CLI
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_ACCT_654654631253_ECR }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_ACCT_654654631253_ECR }}
+          aws-region: us-east-1
+
+      - name: Login to Amazon ECR
+        run: |
+          aws ecr-public get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin public.ecr.aws/u5x5h6w3
+
+      - name: Build and push Docker images
+        run: |
+              dir=base-images/amazonlinux-base/
+              echo "Building an image present in $dir"
+              image_name=nextflow-approved/public
+              tag_name=$(basename "$dir")
+              docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir
+              docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name
+              echo "Built an image with name --> $image_name:$tag_name"
+
+              for dir in containers/nextflow-base-images/*/;do
+                  echo "Building an image present in $dir"
+                  image_name=nextflow-approved/public
+                  tag_name=$(basename "$dir")
+                  docker build -t public.ecr.aws/u5x5h6w3/$image_name:$tag_name $dir
+                  docker push public.ecr.aws/u5x5h6w3/$image_name:$tag_name
+                  echo "Built an image with name --> $image_name:$tag_name"
+              done
+
+      - name: Trigger `get_layer_info_for_nf_imgs.yml` workflow
+        env:
+          GH_TOKEN: ${{ secrets.PLANXCYBORG_TOKEN }}
+        run: |
+              gh api \
+              --method POST \
+              -H "Accept: application/vnd.github+json" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              /repos/uc-cdis/containers/actions/workflows/get_layer_info_for_nf_imgs.yml/dispatches \
+              -f "ref=master"
+
+      - name: Delete untagged docker images from public ECR
+        run: |
+            REPO_NAME=nextflow-approved/public
+            IMAGE_IDS=$(aws ecr-public describe-images --repository-name $REPO_NAME --query 'imageDetails[?imageTags==null].imageDigest' --output text)
+            if [ -n "$IMAGE_IDS" ]; then
+              for IMAGE_ID in $IMAGE_IDS; do
+                echo "Deleting image with SHA hash - $IMAGE_ID"
+                aws ecr-public batch-delete-image --repository-name $REPO_NAME --image-ids imageDigest=$IMAGE_ID
+              done
+            else
+              echo "No untagged images to delete."
+            fi
diff --git a/.github/workflows/build_and_push_python_image.yml b/.github/workflows/build_and_push_python_image.yml
@@ -1,13 +1,14 @@
-name: Build Python Images and Push to Quay and ECR
+name: Build and Push python3.9-data-science
 
 on:
   push:
     paths:
-    - python3.9-data-science/Dockerfile
+    - python3.9-data-science/**
+    - .github/workflows/build_and_push_python_image.yml
 
 jobs:
   python_3-9:
-    name: Python 3.9 Build and Push
+    name: Build and Push python3.9-data-science
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./python3.9-data-science/Dockerfile"

diff --git a/.github/workflows/build_azlinux_jupyter_scipy_image.yml b/.github/workflows/build_azlinux_jupyter_scipy_image.yml
@@ -0,0 +1,22 @@
+name: Build and push AmazonLinux jupyter-scipy
+
+on:
+  push:
+    paths:
+    - azlinux-jupyter-scipy/**
+    - .github/workflows/build_azlinux_jupyter_scipy_image.yml
+
+jobs:
+  jupyter-nextflow:
+    name: Build and Push AmazonLinux jupyter-scipy
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./azlinux-jupyter-scipy/Dockerfile"
+      DOCKERFILE_BUILD_CONTEXT: "./azlinux-jupyter-scipy"
+      OVERRIDE_REPO_NAME: "jupyter-notebook"
+      OVERRIDE_TAG_NAME: "azlinux-scipy-$(echo ${GITHUB_REF#refs/*/} | tr / _)"
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
diff --git a/.github/workflows/build_brh_notebook_image.yml b/.github/workflows/build_brh_notebook_image.yml
@@ -1,4 +1,4 @@
-name: Build and push BRH notebook image
+name: Build and push BRH tutorial image
 
 on:
   push:
@@ -71,7 +71,7 @@ jobs:
             print(f"None of {changed_brh_notebook_files} triggers a build for any of {subdirs}. Done.")
             exit(0)
           elif len(buildable_images) > 1:
-            print("Found multiple notebook directories with changes: {buildable_images}")
+            print("Found multiple directories with changes: {buildable_images}")
             print("Only one image can be built at a time. Exiting.")
             exit(1)
 
@@ -91,7 +91,7 @@ jobs:
         id: build-image
         uses: redhat-actions/buildah-build@v2
         with:
-          image: brh-notebooks
+          image: BRH-notebooks
           tags:
             ${{ steps.sanitize_name.outputs.image_name }}__${{ steps.extract_branch.outputs.branch }}
             ${{ steps.sanitize_name.outputs.image_name }}__${{ github.sha }}

diff --git a/.github/workflows/build_heal_notebook_image.yml b/.github/workflows/build_heal_notebook_image.yml
@@ -4,11 +4,12 @@ on:
   push:
     paths:
     - HEAL-notebooks/*/**
+    - .github/workflows/build_heal_notebook_image.yml
 
 jobs:
   push-image:
     runs-on: ubuntu-latest
-    steps:  
+    steps:
       - name: Maximize build space
         uses: easimon/maximize-build-space@master
         with:
@@ -17,13 +18,13 @@ jobs:
           remove-dotnet: 'true'
           remove-android: 'true'
           remove-haskell: 'true'
-      
+
       - name: clean
-        run: sudo apt clean    
-      
+        run: sudo apt clean
+
       - name: Extract branch name
         shell: bash
-        run: echo "::set-output name=branch::$(echo $(echo ${GITHUB_REF#refs/heads/} | tr / _))"
+        run: echo "branch=$(echo $(echo ${GITHUB_REF#refs/heads/} | tr  / _))" >> $GITHUB_OUTPUT
         id: extract_branch
 
       - name: Checkout repo
@@ -77,14 +78,15 @@ jobs:
 
           build_target = buildable_images[0]
           print(f"Will trigger build for: {build_target}")
-          print(f"::set-output name=build_target::{build_target}")
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+            print(f'build_target={build_target}', file=fh)
 
       - if: ${{ steps.parse_image.outputs.build_target }}
         name: Sanitize image name
         id: sanitize_name
         run: |
           IMAGE_NAME=$( sed 's/[^[:alnum:]]/_/g' <<< ${{ steps.parse_image.outputs.build_target }} );
-          echo "::set-output name=image_name::$IMAGE_NAME"
+          echo "image_name=$IMAGE_NAME" >> $GITHUB_OUTPUT
 
       - if: ${{ steps.parse_image.outputs.build_target }}
         name: Build image

diff --git a/.github/workflows/build_jupyter_covid19_image.yml b/.github/workflows/build_jupyter_covid19_image.yml
@@ -0,0 +1,22 @@
+name: Build and Push jupyter-covid19
+
+on:
+  push:
+    paths:
+    - jupyter-covid19/**
+    - .github/workflows/build_jupyter_covid19_image.yml
+
+jobs:
+  jupyter-covid19:
+    name: Build and Push jupyter-covid19
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./jupyter-covid19/Dockerfile"
+      DOCKERFILE_BUILD_CONTEXT: "./jupyter-covid19"
+      OVERRIDE_REPO_NAME: "jupyter-notebook"
+      OVERRIDE_TAG_NAME: "covid19-$(echo ${GITHUB_REF#refs/*/} | tr / _)"
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
diff --git a/.github/workflows/build_jupyter_nextflow_image.yml b/.github/workflows/build_jupyter_nextflow_image.yml
@@ -0,0 +1,22 @@
+name: Build and Push Jupyter-Nextflow image
+
+on:
+  push:
+    paths:
+    - jupyter-nextflow/**
+    - .github/workflows/build_jupyter_nextflow_image.yml
+
+jobs:
+  jupyter-nextflow:
+    name: Build and Push jupyter-nextflow image
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./jupyter-nextflow/Dockerfile"
+      DOCKERFILE_BUILD_CONTEXT: "./jupyter-nextflow"
+      OVERRIDE_REPO_NAME: "jupyter-notebook"
+      OVERRIDE_TAG_NAME: "nextflow-$(echo ${GITHUB_REF#refs/*/} | tr / _)"
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
diff --git a/.github/workflows/build_jupyter_restricted_download_image.yml b/.github/workflows/build_jupyter_restricted_download_image.yml
@@ -0,0 +1,22 @@
+name: Build and Push jupyter-restricted-download
+
+on:
+  push:
+    paths:
+    - jupyter-restricted-download/**
+    - .github/workflows/build_jupyter_restricted_download_image.yml
+
+jobs:
+  jupyter-restricted-download:
+    name: Build and Push jupyter-restricted-download
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./jupyter-restricted-download/Dockerfile"
+      DOCKERFILE_BUILD_CONTEXT: "./jupyter-restricted-download"
+      OVERRIDE_REPO_NAME: "jupyter-notebook"
+      OVERRIDE_TAG_NAME: "restricted-download-$(echo ${GITHUB_REF#refs/*/} | tr / _)"
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
diff --git a/.github/workflows/build_push_stata_gen3_licensed.yml b/.github/workflows/build_push_stata_gen3_licensed.yml
@@ -0,0 +1,69 @@
+name: Push Stata Gen3-licensed Image to quay
+
+on:
+  push:
+    paths:
+      - jupyter-pystata-gen3-licensed/**
+      - .github/workflows/build_push_stata_gen3_licensed.yml
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 30000
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+      - uses: actions/checkout@v2
+      - uses: prewk/[email protected]
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          SOURCE: 's3://ctds-stata/Stata17Linux64.tar.gz'
+          DEST: './jupyter-pystata-gen3-licensed/resources/'
+
+      - name: Extract branch name
+        shell: bash
+        run: echo "branch=$(echo $(echo ${GITHUB_REF#refs/*/} | tr / _))" >> $GITHUB_OUTPUT
+        id: extract_branch
+
+      - name: Determine image to build
+        id: parse_image
+        shell: python
+        run: |
+          import os
+
+          build_target = "jupyter-pystata-licensed"
+          print(f"Will trigger build for: {build_target}")
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+            print(f'build_target={build_target}', file=fh)
+
+      - if: ${{ steps.parse_image.outputs.build_target }}
+        name: Sanitize image name
+        id: sanitize_name
+        run: |
+          IMAGE_NAME=$( sed 's/[^[:alnum:]]/_/g' <<< ${{ steps.parse_image.outputs.build_target }} );
+          echo "image_name=$IMAGE_NAME" >> $GITHUB_OUTPUT
+
+      - name: Build Image
+        id: build-image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          image: jupyter-pystata-gen3-licensed
+          tags: ${{ steps.extract_branch.outputs.branch }}
+          dockerfiles: ./jupyter-pystata-gen3-licensed/Dockerfile
+
+      - name: Push To quay.io
+        id: push-to-quay
+        uses: redhat-actions/push-to-registry@v2
+        with:
+          image: ${{ steps.build-image.outputs.image }}
+          tags: ${{ steps.build-image.outputs.tags }}
+          registry: quay.io/cdis
+          username: ${{ secrets.QUAY_SERVICE_ACCOUNT_USER }}
+          password: ${{ secrets.QUAY_SERVICE_ACCOUNT_PASSWORD }}
diff --git a/.github/workflows/push_stata_image.yml → ...kflows/build_push_stata_user_licensed.yml b/.github/workflows/push_stata_image.yml → ...kflows/build_push_stata_user_licensed.yml
@@ -1,10 +1,10 @@
-name: Push Stata Image to quay
+name: Push Stata User-licensed Image to quay
 
 on:
   push:
     paths:
-      - jupyter-pystata/**
-      - .github/workflows/push_stata_image.yml
+      - jupyter-pystata-user-licensed/**
+      - .github/workflows/build_push_stata_user_licensed.yml
   workflow_dispatch:
 
 jobs:
@@ -20,25 +20,25 @@ jobs:
           remove-android: 'true'
           remove-haskell: 'true'
       - uses: actions/checkout@v2
-      - uses: prewk/s3-cp-action@master
+      - uses: prewk/s3-cp-action@v0.1.1
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           SOURCE: 's3://ctds-stata/Stata17Linux64.tar.gz'
-          DEST: './jupyter-pystata/resources/'
+          DEST: './jupyter-pystata-user-licensed/resources/'
 
       - name: Extract branch name
         shell: bash
-        run: echo "::set-output name=branch::$(echo $(echo ${GITHUB_REF#refs/heads/} | tr / _))"
+        run: echo "::set-output name=branch::$(echo $(echo ${GITHUB_REF#refs/*/} | tr / _))"
         id: extract_branch
 
       - name: Build Image
         id: build-image
         uses: redhat-actions/buildah-build@v2
         with:
-          image: stata-heal
-          tags: ${{ steps.extract_branch.outputs.branch }} ${{ github.sha }}
-          dockerfiles: ./jupyter-pystata/Dockerfile
+          image: jupyter-pystata-user-licensed
+          tags: ${{ steps.extract_branch.outputs.branch }}
+          dockerfiles: ./jupyter-pystata-user-licensed/Dockerfile
 
       - name: Push To quay.io
         id: push-to-quay