From ab676c466b4500b6492549ae218ea9c4be943599 Mon Sep 17 00:00:00 2001 From: souheil-yazji Date: Mon, 16 Oct 2023 14:03:53 +0000 Subject: [PATCH 01/36] refactor(docker): clean up #1 --- docker-bits/0_cpu.Dockerfile | 6 ----- docker-bits/2_cpu.Dockerfile | 4 +-- docker-bits/2_tensorflow.Dockerfile | 8 +++--- docker-bits/3_Kubeflow.Dockerfile | 22 +++++---------- output/jupyterlab-cpu/Dockerfile | 28 +++++-------------- output/jupyterlab-pytorch/Dockerfile | 28 +++++-------------- output/jupyterlab-tensorflow/Dockerfile | 36 +++++++------------------ output/remote-desktop/Dockerfile | 22 +++++---------- output/rstudio/Dockerfile | 28 +++++-------------- output/sas/Dockerfile | 22 +++++---------- 10 files changed, 52 insertions(+), 152 deletions(-) diff --git a/docker-bits/0_cpu.Dockerfile b/docker-bits/0_cpu.Dockerfile index 23c989a38..752d89dcc 100644 --- a/docker-bits/0_cpu.Dockerfile +++ b/docker-bits/0_cpu.Dockerfile @@ -14,9 +14,3 @@ RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ && apt-get upgrade --yes libwebp7 \ && rm -rf /var/lib/apt/lists/* - -#updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 -#TODO: Evaluate if this is still necessary when updating the base image -RUN pip install --force-reinstall cryptography==39.0.1 && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER diff --git a/docker-bits/2_cpu.Dockerfile b/docker-bits/2_cpu.Dockerfile index b5e930840..98581e0e4 100644 --- a/docker-bits/2_cpu.Dockerfile +++ b/docker-bits/2_cpu.Dockerfile @@ -1,6 +1,6 @@ # Create conda environment (CPU only) with many useful packages. -RUN conda create -n pycpu --yes \ +RUN mamba create -n pycpu --yes \ python==3.11.0 ipython==8.11.0 sphinx==6.1.3 \ boto==2.49.0 s3fs==2023.3.0 \ dos2unix==7.4.1 parallel==20230122 \ @@ -12,6 +12,6 @@ RUN conda create -n pycpu --yes \ gensim==4.3.0 nltk==3.8.1 spacy==3.5.0 \ pytorch==1.13.1 torchaudio==0.13.1 torchvision==0.14.1 cpuonly==2.0 \ -c pytorch -c conda-forge && \ - conda clean --all -f -y && \ + mamba clean --all -f -y && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/docker-bits/2_tensorflow.Dockerfile b/docker-bits/2_tensorflow.Dockerfile index 6b087f518..4ac789207 100644 --- a/docker-bits/2_tensorflow.Dockerfile +++ b/docker-bits/2_tensorflow.Dockerfile @@ -1,8 +1,8 @@ # Install Tensorflow -RUN pip install --quiet \ - 'tensorflow' \ - 'keras' \ - 'ipykernel==6.21.3' \ +RUN mamba install --quiet --yes \ + tensorflow \ + keras \ + ipykernel \ && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index 5135f201c..d890d908b 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -1,27 +1,17 @@ USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ +RUN mamba --no-cache-dir install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ 'kubeflow-pytorchjob==0.1.3' \ 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index d0c3e8b8d..da387beb2 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -25,43 +25,27 @@ RUN apt-get update --yes \ && apt-get upgrade --yes libwebp7 \ && rm -rf /var/lib/apt/lists/* -#updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 -#TODO: Evaluate if this is still necessary when updating the base image -RUN pip install --force-reinstall cryptography==39.0.1 && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - ############################### ### docker-bits/3_Kubeflow.Dockerfile ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ +RUN mamba --no-cache-dir install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ 'kubeflow-pytorchjob==0.1.3' \ 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index c428f9df4..043243b08 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -25,12 +25,6 @@ RUN apt-get update --yes \ && apt-get upgrade --yes libwebp7 \ && rm -rf /var/lib/apt/lists/* -#updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 -#TODO: Evaluate if this is still necessary when updating the base image -RUN pip install --force-reinstall cryptography==39.0.1 && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - ############################### ### docker-bits/2_pytorch.Dockerfile ############################### @@ -58,32 +52,22 @@ RUN mamba create -n torch && \ ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ +RUN mamba --no-cache-dir install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ 'kubeflow-pytorchjob==0.1.3' \ 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 670557265..ef3d53d92 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -25,12 +25,6 @@ RUN apt-get update --yes \ && apt-get upgrade --yes libwebp7 \ && rm -rf /var/lib/apt/lists/* -#updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 -#TODO: Evaluate if this is still necessary when updating the base image -RUN pip install --force-reinstall cryptography==39.0.1 && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - ############################### ### docker-bits/1_CUDA-11.8.0.Dockerfile ############################### @@ -151,10 +145,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ############################### # Install Tensorflow -RUN pip install --quiet \ - 'tensorflow' \ - 'keras' \ - 'ipykernel==6.21.3' \ +RUN mamba install --quiet --yes \ + tensorflow \ + keras \ + ipykernel \ && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER @@ -164,32 +158,22 @@ RUN pip install --quiet \ ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ +RUN mamba --no-cache-dir install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ 'kubeflow-pytorchjob==0.1.3' \ 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 55af1f51f..7d4cde62c 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -37,32 +37,22 @@ RUN chmod 555 /usr/local/lib/R /usr/local/lib/R/site-library/ ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ +RUN mamba --no-cache-dir install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ 'kubeflow-pytorchjob==0.1.3' \ 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile diff --git a/output/rstudio/Dockerfile b/output/rstudio/Dockerfile index a08092193..6ec209bf3 100644 --- a/output/rstudio/Dockerfile +++ b/output/rstudio/Dockerfile @@ -25,43 +25,27 @@ RUN apt-get update --yes \ && apt-get upgrade --yes libwebp7 \ && rm -rf /var/lib/apt/lists/* -#updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 -#TODO: Evaluate if this is still necessary when updating the base image -RUN pip install --force-reinstall cryptography==39.0.1 && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - ############################### ### docker-bits/3_Kubeflow.Dockerfile ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ +RUN mamba --no-cache-dir install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ 'kubeflow-pytorchjob==0.1.3' \ 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 65c1d0a20..87df17ea4 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -32,32 +32,22 @@ RUN pip install --force-reinstall cryptography==39.0.1 && \ ############################### USER root +COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN pip3 --no-cache-dir install --quiet \ +RUN mamba --no-cache-dir install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ - 'jupyterlab==3.6.1' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ 'kubeflow-pytorchjob==0.1.3' \ 'kubeflow-tfjob==0.1.3' \ - 'minio==7.1.13' \ 'joblib==1.2.0' \ - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN pip3 --no-cache-dir install --quiet \ + # s3 file system tool forked by Zach, ~4 years old, to be upgraded + 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -COPY aaw-suspend-server.sh /usr/local/bin -RUN chmod +x /usr/local/bin/aaw-suspend-server.sh + fix-permissions /home/$NB_USER && \ + chmod +x /usr/local/bin/aaw-suspend-server.sh ############################### ### docker-bits/4_CLI.Dockerfile From 1c8bb7a0a74c4253795eb583a0d7cca30a39a06b Mon Sep 17 00:00:00 2001 From: souheil-yazji Date: Mon, 16 Oct 2023 15:04:43 +0000 Subject: [PATCH 02/36] feat(CI): build on push --- .github/workflows/build_push.yaml | 147 ++++++++++++++---------------- 1 file changed, 69 insertions(+), 78 deletions(-) diff --git a/.github/workflows/build_push.yaml b/.github/workflows/build_push.yaml index d319b47a6..29cdb6383 100644 --- a/.github/workflows/build_push.yaml +++ b/.github/workflows/build_push.yaml @@ -22,20 +22,10 @@ # # 2. Change the values for the REGISTRY_NAME, CLUSTER_NAME, CLUSTER_RESOURCE_GROUP and NAMESPACE environment variables (below in build-push). name: build_and_push -on: - schedule: - # Execute at 2am EST every day - - cron: '0 21 * * *' - push: - branches: - - 'master' - pull_request: - types: - - 'opened' - - 'synchronize' - - 'reopened' -env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} +on: push + # schedule: + # # Execute at 2am EST every day + # - cron: '0 21 * * *' jobs: # Any checks that run pre-build @@ -62,7 +52,6 @@ jobs: CLUSTER_RESOURCE_GROUP: k8s-cancentral-01-covid-aks LOCAL_REPO: localhost:5000 TRIVY_VERSION: "v0.31.3" - HADOLINT_VERSION: "2.12.0" strategy: fail-fast: false matrix: @@ -83,18 +72,18 @@ jobs: ports: - 5000:5000 steps: - - name: Set ENV variables for a PR containing the auto-deploy tag - if: github.event_name == 'pull_request' && contains( github.event.pull_request.labels.*.name, 'auto-deploy') - run: | - echo "REGISTRY=k8scc01covidacrdev.azurecr.io" >> "$GITHUB_ENV" - echo "IMAGE_VERSION=dev" >> "$GITHUB_ENV" + # - name: Set ENV variables for a PR containing the auto-deploy tag + # if: github.event_name == 'pull_request' && contains( github.event.pull_request.labels.*.name, 'auto-deploy') + # run: | + # echo "REGISTRY=k8scc01covidacrdev.azurecr.io" >> "$GITHUB_ENV" + # echo "IMAGE_VERSION=dev" >> "$GITHUB_ENV" - - name: Set ENV variables for pushes to master - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - run: | - echo "REGISTRY=k8scc01covidacr.azurecr.io" >> "$GITHUB_ENV" - echo "IMAGE_VERSION=v1" >> "$GITHUB_ENV" - echo "IS_LATEST=true" >> "$GITHUB_ENV" + # - name: Set ENV variables for pushes to master + # if: github.event_name == 'push' && github.ref == 'refs/heads/master' + # run: | + # echo "REGISTRY=k8scc01covidacr.azurecr.io" >> "$GITHUB_ENV" + # echo "IMAGE_VERSION=v1" >> "$GITHUB_ENV" + # echo "IS_LATEST=true" >> "$GITHUB_ENV" - uses: actions/checkout@master @@ -111,32 +100,31 @@ jobs: id: notebook-name shell: bash run: | - echo NOTEBOOK_NAME=${{ matrix.notebook }} >> $GITHUB_OUTPUT + echo name=NOTEBOOK_NAME${{ matrix.notebook }} >> $GITHUB_OUTPUT # Connect to Azure Container registry (ACR) - - uses: azure/docker-login@v1 - with: - login-server: ${{ env.REGISTRY_NAME }}.azurecr.io - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_PASSWORD }} + # - uses: azure/docker-login@v1 + # with: + # login-server: ${{ env.REGISTRY_NAME }}.azurecr.io + # username: ${{ secrets.REGISTRY_USERNAME }} + # password: ${{ secrets.REGISTRY_PASSWORD }} - # Connect to Azure DEV Container registry (ACR) - - uses: azure/docker-login@v1 - with: - login-server: ${{ env.DEV_REGISTRY_NAME }}.azurecr.io - username: ${{ secrets.DEV_REGISTRY_USERNAME }} - password: ${{ secrets.DEV_REGISTRY_PASSWORD }} + # # Connect to Azure DEV Container registry (ACR) + # - uses: azure/docker-login@v1 + # with: + # login-server: ${{ env.DEV_REGISTRY_NAME }}.azurecr.io + # username: ${{ secrets.DEV_REGISTRY_USERNAME }} + # password: ${{ secrets.DEV_REGISTRY_PASSWORD }} # Image building/storing locally - name: Make Dockerfiles run: make generate-dockerfiles - - name: Run Hadolint - run: | - sudo curl -L https://github.com/hadolint/hadolint/releases/download/v${{ env.HADOLINT_VERSION }}/hadolint-Linux-x86_64 --output hadolint - sudo chmod +x hadolint - ./hadolint output/${{ matrix.notebook }}/Dockerfile --no-fail - + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: --debug + # make build emits full_image_name, image_tag, and image_repo outputs - name: Build image id: build-image @@ -150,6 +138,7 @@ jobs: - name: Push image to local registry (default pushes all tags) run: make push/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }} + # Image testing - name: Set Up Python for Test Suite @@ -165,6 +154,8 @@ jobs: - name: Test image run: make test/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }} + # Image scanning + # Free up space from build process (containerscan action will run out of space if we don't) - run: ./.github/scripts/cleanup_runner.sh @@ -173,43 +164,43 @@ jobs: # see https://github.com/StatCan/aaw-private/issues/11 -- should be re-enabled if: steps.notebook-name.outputs.NOTEBOOK_NAME != 'sas' run: | - printf ${{ secrets.CVE_ALLOWLIST }} > .trivyignore curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin ${{ env.TRIVY_VERSION }} trivy image ${{ steps.build-image.outputs.full_image_name }} --exit-code 1 --timeout=20m --security-checks vuln --severity CRITICAL + # Push image to ACR # Pushes if this is a push to master or an update to a PR that has auto-deploy label - - name: Test if we should push to ACR - id: should-i-push - if: | - github.event_name == 'push' || - ( - github.event_name == 'pull_request' && - contains( github.event.pull_request.labels.*.name, 'auto-deploy') - ) - run: echo 'boolean=true' >> $GITHUB_OUTPUT - - # Pull the local image back, then "build" it (will just tag the pulled image) - - name: Pull image back from local repo - if: steps.should-i-push.outputs.boolean == 'true' - run: docker pull ${{ steps.build-image.outputs.full_image_name }} - - # Rename the localhost:5000/imagename:tag built above to use the real repo - # (get above's name from build-image's output) - - name: Tag images with real repository - if: steps.should-i-push.outputs.boolean == 'true' - run: > - make post-build/${{ matrix.notebook }} DEFAULT_REPO=$REGISTRY IS_LATEST=$IS_LATEST - IMAGE_VERSION=$IMAGE_VERSION SOURCE_FULL_IMAGE_NAME=${{ steps.build-image.outputs.full_image_name }} - - - name: Push image to registry - if: steps.should-i-push.outputs.boolean == 'true' - run: | - make push/${{ matrix.notebook }} DEFAULT_REPO=$REGISTRY + # - name: Test if we should push to ACR + # id: should-i-push + # if: | + # github.event_name == 'push' || + # ( + # github.event_name == 'pull_request' && + # contains( github.event.pull_request.labels.*.name, 'auto-deploy') + # ) + # run: echo "::set-output name=boolean::true" + + # # Pull the local image back, then "build" it (will just tag the pulled image) + # - name: Pull image back from local repo + # if: steps.should-i-push.outputs.boolean == 'true' + # run: docker pull ${{ steps.build-image.outputs.full_image_name }} + + # # Rename the localhost:5000/imagename:tag built above to use the real repo + # # (get above's name from build-image's output) + # - name: Tag images with real repository + # if: steps.should-i-push.outputs.boolean == 'true' + # run: > + # make post-build/${{ matrix.notebook }} DEFAULT_REPO=$REGISTRY IS_LATEST=$IS_LATEST + # IMAGE_VERSION=$IMAGE_VERSION SOURCE_FULL_IMAGE_NAME=${{ steps.build-image.outputs.full_image_name }} + + # - name: Push image to registry + # if: steps.should-i-push.outputs.boolean == 'true' + # run: | + # make push/${{ matrix.notebook }} DEFAULT_REPO=$REGISTRY - - name: Slack Notification - if: failure() && github.event_name=='schedule' - uses: act10ns/slack@v1 - with: - status: failure - message: Build failed. https://github.com/StatCan/aaw-kubeflow-containers/actions/runs/${{github.run_id}} + # - name: Slack Notification + # if: failure() && github.event_name=='schedule' + # uses: act10ns/slack@v1 + # with: + # status: failure + # message: Build failed. https://github.com/StatCan/aaw-kubeflow-containers/actions/runs/${{github.run_id}} From c5bb76b7cc15570fb2df4606b43f7bc7472af44d Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 11:17:17 -0400 Subject: [PATCH 03/36] fix(kubeflow): mamba no cache --- docker-bits/3_Kubeflow.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index d890d908b..a32437bf2 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -2,7 +2,7 @@ USER root COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN mamba --no-cache-dir install --quiet \ +RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ From bb93704ba069ab3c8561e614004be3d4df9228e4 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 11:37:48 -0400 Subject: [PATCH 04/36] fix(kubectl): update kubectl --- docker-bits/4_CLI.Dockerfile | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docker-bits/4_CLI.Dockerfile b/docker-bits/4_CLI.Dockerfile index d22374109..76558ef50 100644 --- a/docker-bits/4_CLI.Dockerfile +++ b/docker-bits/4_CLI.Dockerfile @@ -1,5 +1,8 @@ USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -21,11 +24,10 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb # ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 @@ -40,18 +42,14 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ From 041f93128974fda986c6a1f70044432892d07c41 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 11:38:47 -0400 Subject: [PATCH 05/36] chore(dockerfiles): update output --- output/jupyterlab-cpu/Dockerfile | 26 ++++++++++++------------- output/jupyterlab-pytorch/Dockerfile | 26 ++++++++++++------------- output/jupyterlab-tensorflow/Dockerfile | 26 ++++++++++++------------- output/remote-desktop/Dockerfile | 26 ++++++++++++------------- output/rstudio/Dockerfile | 26 ++++++++++++------------- output/sas/Dockerfile | 26 ++++++++++++------------- 6 files changed, 72 insertions(+), 84 deletions(-) diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index da387beb2..1d3bbb092 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -33,7 +33,7 @@ USER root COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN mamba --no-cache-dir install --quiet \ +RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ @@ -53,6 +53,9 @@ RUN mamba --no-cache-dir install --quiet \ USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -74,11 +77,10 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb # ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 @@ -93,18 +95,14 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index 043243b08..d5e200140 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -55,7 +55,7 @@ USER root COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN mamba --no-cache-dir install --quiet \ +RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ @@ -75,6 +75,9 @@ RUN mamba --no-cache-dir install --quiet \ USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -96,11 +99,10 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb # ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 @@ -115,18 +117,14 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index ef3d53d92..be1d447fb 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -161,7 +161,7 @@ USER root COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN mamba --no-cache-dir install --quiet \ +RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ @@ -181,6 +181,9 @@ RUN mamba --no-cache-dir install --quiet \ USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -202,11 +205,10 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb # ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 @@ -221,18 +223,14 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 7d4cde62c..3088bad66 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -40,7 +40,7 @@ USER root COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN mamba --no-cache-dir install --quiet \ +RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ @@ -60,6 +60,9 @@ RUN mamba --no-cache-dir install --quiet \ USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -81,11 +84,10 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb # ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 @@ -100,18 +102,14 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ diff --git a/output/rstudio/Dockerfile b/output/rstudio/Dockerfile index 6ec209bf3..327a1557a 100644 --- a/output/rstudio/Dockerfile +++ b/output/rstudio/Dockerfile @@ -33,7 +33,7 @@ USER root COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN mamba --no-cache-dir install --quiet \ +RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ @@ -53,6 +53,9 @@ RUN mamba --no-cache-dir install --quiet \ USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -74,11 +77,10 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb # ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 @@ -93,18 +95,14 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 87df17ea4..0615fc728 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -35,7 +35,7 @@ USER root COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 -RUN mamba --no-cache-dir install --quiet \ +RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ @@ -55,6 +55,9 @@ RUN mamba --no-cache-dir install --quiet \ USER root +# Add helpers for shell initialization +COPY shell_helpers.sh /tmp/shell_helpers.sh + # Dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -76,11 +79,10 @@ RUN apt-get update && \ && \ rm -rf /var/lib/apt/lists/* -COPY --from=minio/mc:RELEASE.2022-03-17T20-25-06Z /bin/mc /usr/local/bin/mc-original -ARG KUBECTL_VERSION=v1.15.10 -ARG KUBECTL_URL=https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -ARG KUBECTL_SHA=38a0f73464f1c39ca383fd43196f84bdbe6e553fe3e677b6e7012ef7ad5eaf2b +ARG KUBECTL_VERSION=v1.28.2 +ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb # ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 @@ -95,18 +97,14 @@ ARG ARGO_CLI_VERSION=v3.4.5 ARG ARGO_CLI_URL=https://github.com/argoproj/argo-workflows/releases/download/${ARGO_CLI_VERSION}/argo-linux-amd64.gz ARG ARGO_CLI_SHA=0528ff0c0aa87a3f150376eee2f1b26e8b41eb96578c43d715c906304627d3a1 -# Add helpers for shell initialization -COPY shell_helpers.sh /tmp/shell_helpers.sh - -# Install OpenJDK-8 -RUN apt-get update && \ +RUN \ + # OpenJDK-8 + apt-get update && \ apt-get install -y openjdk-8-jre && \ apt-get clean && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - - -RUN \ + fix-permissions /home/$NB_USER \ + && \ # kubectl curl -LO "${KUBECTL_URL}" \ && echo "${KUBECTL_SHA} kubectl" | sha256sum -c - \ From 2c7d00f7db36eec90e36bc4e4b7ddf54b0e9e253 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 12:18:39 -0400 Subject: [PATCH 06/36] refactor(kubeflow): remove deprecated python SDKs --- docker-bits/3_Kubeflow.Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index a32437bf2..c1a445139 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -6,8 +6,6 @@ RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ 'PyYAML==6.0' \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ From 772b1f0edddf219761129f7b7423b2d5b2fd5ddd Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 12:19:10 -0400 Subject: [PATCH 07/36] refactor(kubeflow): update PyYAML --- docker-bits/3_Kubeflow.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index c1a445139..2c2940871 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -5,7 +5,7 @@ COPY aaw-suspend-server.sh /usr/local/bin RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'notebook==6.5.3' \ - 'PyYAML==6.0' \ + 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ From 55fc90ef9999c643e2cd317b7a281a4167697d2e Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 12:19:34 -0400 Subject: [PATCH 08/36] refactor(kubeflow): remove notebook, comes from upstream --- docker-bits/3_Kubeflow.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index 2c2940871..38ea91b7f 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -4,7 +4,6 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded From 2b3a916615832e955cc6100488c62bdfbade9cd1 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 12:24:31 -0400 Subject: [PATCH 09/36] refactor(kubeflow): replace s3fs fork with upstream --- docker-bits/3_Kubeflow.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index 38ea91b7f..f9cd438a3 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -7,7 +7,7 @@ RUN mamba install --quiet \ 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ + 's3fs' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ From be6d01032d9615a71d179ae1c30eb53f18ac75ad Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 12:27:31 -0400 Subject: [PATCH 10/36] refactor(kubeflow): remove upstream utils --- docker-bits/4_CLI.Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-bits/4_CLI.Dockerfile b/docker-bits/4_CLI.Dockerfile index 76558ef50..8dc653498 100644 --- a/docker-bits/4_CLI.Dockerfile +++ b/docker-bits/4_CLI.Dockerfile @@ -9,7 +9,6 @@ RUN apt-get update && \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -18,8 +17,6 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* From 12cd2d538f7f00745e4d462a17391f222921f70b Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 12:28:26 -0400 Subject: [PATCH 11/36] refactor(kubeflow): add kubeflow training py sdk --- docker-bits/3_Kubeflow.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index f9cd438a3..63809d97e 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -6,6 +6,7 @@ RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ + 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ 'fire==0.5.0' && \ From 78cf39cc3571a4c4c33a8471c766c50d3a58eae2 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 12:29:19 -0400 Subject: [PATCH 12/36] chore(dockerfiles): update output --- output/jupyterlab-cpu/Dockerfile | 11 +++-------- output/jupyterlab-pytorch/Dockerfile | 11 +++-------- output/jupyterlab-tensorflow/Dockerfile | 11 +++-------- output/remote-desktop/Dockerfile | 11 +++-------- output/rstudio/Dockerfile | 11 +++-------- output/sas/Dockerfile | 11 +++-------- 6 files changed, 18 insertions(+), 48 deletions(-) diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 1d3bbb092..2388e07d7 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -35,13 +35,11 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ + 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ + 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ + 's3fs' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ @@ -62,7 +60,6 @@ RUN apt-get update && \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -71,8 +68,6 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index d5e200140..14024e62a 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -57,13 +57,11 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ + 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ + 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ + 's3fs' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ @@ -84,7 +82,6 @@ RUN apt-get update && \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -93,8 +90,6 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index be1d447fb..a9709e069 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -163,13 +163,11 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ + 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ + 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ + 's3fs' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ @@ -190,7 +188,6 @@ RUN apt-get update && \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -199,8 +196,6 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 3088bad66..8a4da2501 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -42,13 +42,11 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ + 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ + 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ + 's3fs' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ @@ -69,7 +67,6 @@ RUN apt-get update && \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -78,8 +75,6 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* diff --git a/output/rstudio/Dockerfile b/output/rstudio/Dockerfile index 327a1557a..e254802ef 100644 --- a/output/rstudio/Dockerfile +++ b/output/rstudio/Dockerfile @@ -35,13 +35,11 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ + 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ + 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ + 's3fs' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ @@ -62,7 +60,6 @@ RUN apt-get update && \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -71,8 +68,6 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 0615fc728..0d9d802e3 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -37,13 +37,11 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ 'Pillow==9.4.0' \ - 'notebook==6.5.3' \ - 'PyYAML==6.0' \ - 'kubeflow-pytorchjob==0.1.3' \ - 'kubeflow-tfjob==0.1.3' \ + 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ + 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded - 'git+https://github.com/zachomedia/s3fs@8aa929f78666ff9e323cde7d9be9262db5a17985' \ + 's3fs' \ 'fire==0.5.0' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ @@ -64,7 +62,6 @@ RUN apt-get update && \ 'byobu' \ 'htop' \ 'jq' \ - 'less' \ 'openssl' \ 'ranger' \ 'tig' \ @@ -73,8 +70,6 @@ RUN apt-get update && \ 'vim' \ 'zip' \ 'zsh' \ - 'wget' \ - 'curl' \ 'dos2unix' \ && \ rm -rf /var/lib/apt/lists/* From 2e1cbf7b55e946d8f60a829c32b65127e833368b Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 12:43:55 -0400 Subject: [PATCH 13/36] refactor(kubeflow): use pip for kf-training --- docker-bits/3_Kubeflow.Dockerfile | 2 +- output/jupyterlab-cpu/Dockerfile | 2 +- output/jupyterlab-pytorch/Dockerfile | 2 +- output/jupyterlab-tensorflow/Dockerfile | 2 +- output/remote-desktop/Dockerfile | 2 +- output/rstudio/Dockerfile | 2 +- output/sas/Dockerfile | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index 63809d97e..f37dd6e7a 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -6,10 +6,10 @@ RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ - 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ 'fire==0.5.0' && \ + pip install 'kubeflow-training' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 2388e07d7..078942ba8 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -37,10 +37,10 @@ RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ - 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ 'fire==0.5.0' && \ + pip install 'kubeflow-training' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index 14024e62a..0fd251199 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -59,10 +59,10 @@ RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ - 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ 'fire==0.5.0' && \ + pip install 'kubeflow-training' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index a9709e069..6f2f654f1 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -165,10 +165,10 @@ RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ - 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ 'fire==0.5.0' && \ + pip install 'kubeflow-training' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 8a4da2501..20a10f615 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -44,10 +44,10 @@ RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ - 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ 'fire==0.5.0' && \ + pip install 'kubeflow-training' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh diff --git a/output/rstudio/Dockerfile b/output/rstudio/Dockerfile index e254802ef..57afceebb 100644 --- a/output/rstudio/Dockerfile +++ b/output/rstudio/Dockerfile @@ -37,10 +37,10 @@ RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ - 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ 'fire==0.5.0' && \ + pip install 'kubeflow-training' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 0d9d802e3..8f77fc3cf 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -39,10 +39,10 @@ RUN mamba install --quiet \ 'Pillow==9.4.0' \ 'PyYAML==6.0.1' \ 'joblib==1.2.0' \ - 'kubeflow-training' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ 'fire==0.5.0' && \ + pip install 'kubeflow-training' && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh From c918a0782e70e065fedae9dcb22652b5e8557b94 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 13:19:24 -0400 Subject: [PATCH 14/36] fix(remote-desktop): include minimamba --- docker-bits/0_Rocker.Dockerfile | 68 ++++++++++++++++++++++++++++++-- output/remote-desktop/Dockerfile | 68 ++++++++++++++++++++++++++++++-- 2 files changed, 130 insertions(+), 6 deletions(-) diff --git a/docker-bits/0_Rocker.Dockerfile b/docker-bits/0_Rocker.Dockerfile index 34493e037..ce1fe4553 100644 --- a/docker-bits/0_Rocker.Dockerfile +++ b/docker-bits/0_Rocker.Dockerfile @@ -4,10 +4,12 @@ FROM rocker/geospatial:4.2.1@sha256:5caca36b8962233f8636540b7c349d3f493f09e864b6e278cb46946ccf60d4d2 # For compatibility with docker stacks -ARG NB_USER="jovyan" ARG HOME=/home/$NB_USER -ENV NB_UID="1000" -ENV NB_GID="100" +ENV NB_UID="1000" \ + NB_GID="100" \ + CONDA_DIR=/opt/conda \ + PATH=$PATH:/opt/conda/bin \ + NB_USER="jovyan" USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" @@ -26,3 +28,63 @@ RUN /rocker_scripts/install_shiny_server.sh \ # Users should install R packages in their home directory RUN chmod 555 /usr/local/lib/R /usr/local/lib/R/site-library/ + + + +# ARG CONDA_VERSION=py38_4.10.3 +# ARG CONDA_MD5=14da4a9a44b337f7ccb8363537f65b9c +ARG PYTHON_VERSION=3.11 + +# #Install Miniconda +# #Has to be appended, else messes with qgis +# RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh -O miniconda.sh && \ +# echo "${CONDA_MD5} miniconda.sh" > miniconda.md5 && \ +# if ! md5sum --status -c miniconda.md5; then exit 1; fi && \ +# mkdir -p /opt && \ +# sh miniconda.sh -b -p /opt/conda && \ +# rm miniconda.sh miniconda.md5 && \ +# ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ +# echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ +# echo "conda activate base" >> ~/.bashrc && \ +# find /opt/conda/ -follow -type f -name '*.a' -delete && \ +# find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ +# /opt/conda/bin/conda clean -afy && \ +# chown -R $NB_UID:$NB_GID /opt/conda +# Download and install Micromamba, and initialize Conda prefix. +# +# Similar projects using Micromamba: +# - Micromamba-Docker: +# - repo2docker: +# Install Python, Mamba and jupyter_core +# Cleanup temporary files and remove Micromamba +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" +WORKDIR /tmp +RUN set -x && \ + arch=$(uname -m) && \ + if [ "${arch}" = "x86_64" ]; then \ + # Should be simpler, see + arch="64"; \ + fi && \ + wget --progress=dot:giga -O /tmp/micromamba.tar.bz2 \ + "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ + tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ + rm /tmp/micromamba.tar.bz2 && \ + PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ + if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ + # Install the packages + ./micromamba install \ + --root-prefix="${CONDA_DIR}" \ + --prefix="${CONDA_DIR}" \ + --yes \ + "${PYTHON_SPECIFIER}" \ + 'mamba' \ + 'jupyter_core' && \ + rm micromamba && \ + # Pin major.minor version of python + mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" \ No newline at end of file diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 20a10f615..4a26018fd 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -9,10 +9,12 @@ FROM rocker/geospatial:4.2.1@sha256:5caca36b8962233f8636540b7c349d3f493f09e864b6e278cb46946ccf60d4d2 # For compatibility with docker stacks -ARG NB_USER="jovyan" ARG HOME=/home/$NB_USER -ENV NB_UID="1000" -ENV NB_GID="100" +ENV NB_UID="1000" \ + NB_GID="100" \ + CONDA_DIR=/opt/conda \ + PATH=$PATH:/opt/conda/bin \ + NB_USER="jovyan" USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" @@ -32,6 +34,66 @@ RUN /rocker_scripts/install_shiny_server.sh \ # Users should install R packages in their home directory RUN chmod 555 /usr/local/lib/R /usr/local/lib/R/site-library/ + + +# ARG CONDA_VERSION=py38_4.10.3 +# ARG CONDA_MD5=14da4a9a44b337f7ccb8363537f65b9c +ARG PYTHON_VERSION=3.11 + +# #Install Miniconda +# #Has to be appended, else messes with qgis +# RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh -O miniconda.sh && \ +# echo "${CONDA_MD5} miniconda.sh" > miniconda.md5 && \ +# if ! md5sum --status -c miniconda.md5; then exit 1; fi && \ +# mkdir -p /opt && \ +# sh miniconda.sh -b -p /opt/conda && \ +# rm miniconda.sh miniconda.md5 && \ +# ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ +# echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ +# echo "conda activate base" >> ~/.bashrc && \ +# find /opt/conda/ -follow -type f -name '*.a' -delete && \ +# find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ +# /opt/conda/bin/conda clean -afy && \ +# chown -R $NB_UID:$NB_GID /opt/conda +# Download and install Micromamba, and initialize Conda prefix. +# +# Similar projects using Micromamba: +# - Micromamba-Docker: +# - repo2docker: +# Install Python, Mamba and jupyter_core +# Cleanup temporary files and remove Micromamba +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" +WORKDIR /tmp +RUN set -x && \ + arch=$(uname -m) && \ + if [ "${arch}" = "x86_64" ]; then \ + # Should be simpler, see + arch="64"; \ + fi && \ + wget --progress=dot:giga -O /tmp/micromamba.tar.bz2 \ + "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ + tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ + rm /tmp/micromamba.tar.bz2 && \ + PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ + if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ + # Install the packages + ./micromamba install \ + --root-prefix="${CONDA_DIR}" \ + --prefix="${CONDA_DIR}" \ + --yes \ + "${PYTHON_SPECIFIER}" \ + 'mamba' \ + 'jupyter_core' && \ + rm micromamba && \ + # Pin major.minor version of python + mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + ############################### ### docker-bits/3_Kubeflow.Dockerfile ############################### From 536174ba3255e2131a66ed3d4af2b2789baaa491 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 13:30:34 -0400 Subject: [PATCH 15/36] fix(remote-desktop): update condarc --- docker-bits/0_Rocker.Dockerfile | 2 +- output/remote-desktop/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-bits/0_Rocker.Dockerfile b/docker-bits/0_Rocker.Dockerfile index ce1fe4553..69dabfd59 100644 --- a/docker-bits/0_Rocker.Dockerfile +++ b/docker-bits/0_Rocker.Dockerfile @@ -60,7 +60,7 @@ ARG PYTHON_VERSION=3.11 # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change -COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" +COPY --chown="${NB_UID}:${NB_GID}" .condarc "${CONDA_DIR}/.condarc" WORKDIR /tmp RUN set -x && \ arch=$(uname -m) && \ diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 4a26018fd..51b3bc982 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -65,7 +65,7 @@ ARG PYTHON_VERSION=3.11 # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change -COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" +COPY --chown="${NB_UID}:${NB_GID}" .condarc "${CONDA_DIR}/.condarc" WORKDIR /tmp RUN set -x && \ arch=$(uname -m) && \ From cac2998e0a1bd2a4fedcfefc68d2615fd394d958 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 14:04:54 -0400 Subject: [PATCH 16/36] fix(remote-desktop): remove prelim condarc --- docker-bits/0_Rocker.Dockerfile | 2 +- docker-bits/6_remote-desktop.Dockerfile | 20 -------------------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/docker-bits/0_Rocker.Dockerfile b/docker-bits/0_Rocker.Dockerfile index 69dabfd59..9702decc3 100644 --- a/docker-bits/0_Rocker.Dockerfile +++ b/docker-bits/0_Rocker.Dockerfile @@ -50,6 +50,7 @@ ARG PYTHON_VERSION=3.11 # find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ # /opt/conda/bin/conda clean -afy && \ # chown -R $NB_UID:$NB_GID /opt/conda +# # Download and install Micromamba, and initialize Conda prefix. # # Similar projects using Micromamba: @@ -60,7 +61,6 @@ ARG PYTHON_VERSION=3.11 # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change -COPY --chown="${NB_UID}:${NB_GID}" .condarc "${CONDA_DIR}/.condarc" WORKDIR /tmp RUN set -x && \ arch=$(uname -m) && \ diff --git a/docker-bits/6_remote-desktop.Dockerfile b/docker-bits/6_remote-desktop.Dockerfile index b3a9d7531..ad5e9d9ca 100644 --- a/docker-bits/6_remote-desktop.Dockerfile +++ b/docker-bits/6_remote-desktop.Dockerfile @@ -339,26 +339,6 @@ RUN apt-get update && apt-get install --yes websockify \ && cp /usr/lib/websockify/rebind.cpython-38-x86_64-linux-gnu.so /usr/lib/websockify/rebind.so \ && clean-layer.sh -#Install Miniconda -#Has to be appended, else messes with qgis -ENV PATH $PATH:/opt/conda/bin - -ARG CONDA_VERSION=py38_4.10.3 -ARG CONDA_MD5=14da4a9a44b337f7ccb8363537f65b9c - -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh -O miniconda.sh && \ - echo "${CONDA_MD5} miniconda.sh" > miniconda.md5 && \ - if ! md5sum --status -c miniconda.md5; then exit 1; fi && \ - mkdir -p /opt && \ - sh miniconda.sh -b -p /opt/conda && \ - rm miniconda.sh miniconda.md5 && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc && \ - find /opt/conda/ -follow -type f -name '*.a' -delete && \ - find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ - /opt/conda/bin/conda clean -afy && \ - chown -R $NB_UID:$NB_GID /opt/conda #Set Defaults ENV HOME=/home/$NB_USER From a482621633daffb964d3f4767b6b49976b54f165 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 14:05:11 -0400 Subject: [PATCH 17/36] chore(dockerfiles): update output --- output/remote-desktop/Dockerfile | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 51b3bc982..20de4c8c3 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -55,6 +55,7 @@ ARG PYTHON_VERSION=3.11 # find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ # /opt/conda/bin/conda clean -afy && \ # chown -R $NB_UID:$NB_GID /opt/conda +# # Download and install Micromamba, and initialize Conda prefix. # # Similar projects using Micromamba: @@ -65,7 +66,6 @@ ARG PYTHON_VERSION=3.11 # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change -COPY --chown="${NB_UID}:${NB_GID}" .condarc "${CONDA_DIR}/.condarc" WORKDIR /tmp RUN set -x && \ arch=$(uname -m) && \ @@ -544,26 +544,6 @@ RUN apt-get update && apt-get install --yes websockify \ && cp /usr/lib/websockify/rebind.cpython-38-x86_64-linux-gnu.so /usr/lib/websockify/rebind.so \ && clean-layer.sh -#Install Miniconda -#Has to be appended, else messes with qgis -ENV PATH $PATH:/opt/conda/bin - -ARG CONDA_VERSION=py38_4.10.3 -ARG CONDA_MD5=14da4a9a44b337f7ccb8363537f65b9c - -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh -O miniconda.sh && \ - echo "${CONDA_MD5} miniconda.sh" > miniconda.md5 && \ - if ! md5sum --status -c miniconda.md5; then exit 1; fi && \ - mkdir -p /opt && \ - sh miniconda.sh -b -p /opt/conda && \ - rm miniconda.sh miniconda.md5 && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc && \ - find /opt/conda/ -follow -type f -name '*.a' -delete && \ - find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ - /opt/conda/bin/conda clean -afy && \ - chown -R $NB_UID:$NB_GID /opt/conda #Set Defaults ENV HOME=/home/$NB_USER From ef8212d2faa5050c2e50c0867bac9d9bf70ea499 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 14:17:50 -0400 Subject: [PATCH 18/36] fix(remote-desktop): add conda forge channel --- docker-bits/0_Rocker.Dockerfile | 15 +++++++++++---- docker-bits/4_CLI.Dockerfile | 5 ++--- output/jupyterlab-cpu/Dockerfile | 5 ++--- output/jupyterlab-pytorch/Dockerfile | 5 ++--- output/jupyterlab-tensorflow/Dockerfile | 5 ++--- output/remote-desktop/Dockerfile | 20 +++++++++++++------- output/rstudio/Dockerfile | 5 ++--- output/sas/Dockerfile | 5 ++--- 8 files changed, 36 insertions(+), 29 deletions(-) diff --git a/docker-bits/0_Rocker.Dockerfile b/docker-bits/0_Rocker.Dockerfile index 9702decc3..6423622cc 100644 --- a/docker-bits/0_Rocker.Dockerfile +++ b/docker-bits/0_Rocker.Dockerfile @@ -5,11 +5,17 @@ FROM rocker/geospatial:4.2.1@sha256:5caca36b8962233f8636540b7c349d3f493f09e864b6 # For compatibility with docker stacks ARG HOME=/home/$NB_USER -ENV NB_UID="1000" \ - NB_GID="100" \ +ARG NB_USER="jovyan" +ARG NB_UID="1000" +ARG NB_GID="100" + +ENV NB_USER="${NB_USER}" \ + NB_UID=${NB_UID} \ + NB_GID=${NB_GID} \ CONDA_DIR=/opt/conda \ PATH=$PATH:/opt/conda/bin \ - NB_USER="jovyan" + NB_USER="jovyan" \ + HOME="/home/${NB_USER}" USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" @@ -81,7 +87,8 @@ RUN set -x && \ --yes \ "${PYTHON_SPECIFIER}" \ 'mamba' \ - 'jupyter_core' && \ + 'jupyter_core' \ + -c conda-forge && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ diff --git a/docker-bits/4_CLI.Dockerfile b/docker-bits/4_CLI.Dockerfile index 8dc653498..1873c0c90 100644 --- a/docker-bits/4_CLI.Dockerfile +++ b/docker-bits/4_CLI.Dockerfile @@ -27,7 +27,6 @@ ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kub ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -53,8 +52,8 @@ RUN \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 078942ba8..39c435313 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -78,7 +78,6 @@ ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kub ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -104,8 +103,8 @@ RUN \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index 0fd251199..a63a6556c 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -100,7 +100,6 @@ ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kub ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -126,8 +125,8 @@ RUN \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 6f2f654f1..1c6ed8a3b 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -206,7 +206,6 @@ ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kub ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -232,8 +231,8 @@ RUN \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 20de4c8c3..e02eec04a 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -10,11 +10,17 @@ FROM rocker/geospatial:4.2.1@sha256:5caca36b8962233f8636540b7c349d3f493f09e864b6 # For compatibility with docker stacks ARG HOME=/home/$NB_USER -ENV NB_UID="1000" \ - NB_GID="100" \ +ARG NB_USER="jovyan" +ARG NB_UID="1000" +ARG NB_GID="100" + +ENV NB_USER="${NB_USER}" \ + NB_UID=${NB_UID} \ + NB_GID=${NB_GID} \ CONDA_DIR=/opt/conda \ PATH=$PATH:/opt/conda/bin \ - NB_USER="jovyan" + NB_USER="jovyan" \ + HOME="/home/${NB_USER}" USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" @@ -86,7 +92,8 @@ RUN set -x && \ --yes \ "${PYTHON_SPECIFIER}" \ 'mamba' \ - 'jupyter_core' && \ + 'jupyter_core' \ + -c conda-forge && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ @@ -147,7 +154,6 @@ ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kub ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -173,8 +179,8 @@ RUN \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ diff --git a/output/rstudio/Dockerfile b/output/rstudio/Dockerfile index 57afceebb..945829497 100644 --- a/output/rstudio/Dockerfile +++ b/output/rstudio/Dockerfile @@ -78,7 +78,6 @@ ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kub ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -104,8 +103,8 @@ RUN \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 8f77fc3cf..613ef9eed 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -80,7 +80,6 @@ ARG KUBECTL_URL=https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kub ARG KUBECTL_SHA=c922440b043e5de1afa3c1382f8c663a25f055978cbc6e8423493ec157579ec5 ARG AZCLI_URL=https://aka.ms/InstallAzureCLIDeb -# ARG AZCLI_SHA=53184ff0e5f73a153dddc2cc7a13897022e7d700153f075724b108a04dcec078 ARG OH_MY_ZSH_URL=https://raw.githubusercontent.com/loket/oh-my-zsh/feature/batch-mode/tools/install.sh ARG OH_MY_ZSH_SHA=22811faf34455a5aeaba6f6b36f2c79a0a454a74c8b4ea9c0760d1b2d7022b03 @@ -106,8 +105,8 @@ RUN \ && chmod +x ./kubectl \ && sudo mv ./kubectl /usr/local/bin/kubectl \ && \ - # AzureCLI - curl -sLO https://aka.ms/InstallAzureCLIDeb \ + # AzureCLI - installation script from Azure + curl -sLO "${AZCLI_URL}" \ && bash InstallAzureCLIDeb \ && rm InstallAzureCLIDeb \ && echo "azcli: ok" \ From d24f1da60e12925c86a15425cfdb8d9eb87b004f Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 14:31:16 -0400 Subject: [PATCH 19/36] fix(remote-desktop): add init condarc --- docker-bits/0_Rocker.Dockerfile | 2 +- output/remote-desktop/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-bits/0_Rocker.Dockerfile b/docker-bits/0_Rocker.Dockerfile index 6423622cc..c3b4abd56 100644 --- a/docker-bits/0_Rocker.Dockerfile +++ b/docker-bits/0_Rocker.Dockerfile @@ -67,6 +67,7 @@ ARG PYTHON_VERSION=3.11 # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change +COPY initial-condarc "${CONDA_DIR}/.condarc" WORKDIR /tmp RUN set -x && \ arch=$(uname -m) && \ @@ -88,7 +89,6 @@ RUN set -x && \ "${PYTHON_SPECIFIER}" \ 'mamba' \ 'jupyter_core' \ - -c conda-forge && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index e02eec04a..07e5c53c7 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -72,6 +72,7 @@ ARG PYTHON_VERSION=3.11 # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change +COPY initial-condarc "${CONDA_DIR}/.condarc" WORKDIR /tmp RUN set -x && \ arch=$(uname -m) && \ @@ -93,7 +94,6 @@ RUN set -x && \ "${PYTHON_SPECIFIER}" \ 'mamba' \ 'jupyter_core' \ - -c conda-forge && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ From 0d3f6741e7fc91a2c085947e206117334b7eff53 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 15:02:21 -0400 Subject: [PATCH 20/36] fix(remote-desktop): add init condarc --- output/remote-desktop/initial-condarc | 6 ++++++ resources/remote-desktop/initial-condarc | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 output/remote-desktop/initial-condarc create mode 100644 resources/remote-desktop/initial-condarc diff --git a/output/remote-desktop/initial-condarc b/output/remote-desktop/initial-condarc new file mode 100644 index 000000000..66ecf6081 --- /dev/null +++ b/output/remote-desktop/initial-condarc @@ -0,0 +1,6 @@ +# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html + +auto_update_conda: false +show_channel_urls: true +channels: + - conda-forge \ No newline at end of file diff --git a/resources/remote-desktop/initial-condarc b/resources/remote-desktop/initial-condarc new file mode 100644 index 000000000..66ecf6081 --- /dev/null +++ b/resources/remote-desktop/initial-condarc @@ -0,0 +1,6 @@ +# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html + +auto_update_conda: false +show_channel_urls: true +channels: + - conda-forge \ No newline at end of file From 5f9c23009c76e139d461459aa03b84547149fc30 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 15:07:56 -0400 Subject: [PATCH 21/36] fix(remote-desktop): fix typo --- docker-bits/0_Rocker.Dockerfile | 2 +- output/remote-desktop/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-bits/0_Rocker.Dockerfile b/docker-bits/0_Rocker.Dockerfile index c3b4abd56..1ba5becaa 100644 --- a/docker-bits/0_Rocker.Dockerfile +++ b/docker-bits/0_Rocker.Dockerfile @@ -88,7 +88,7 @@ RUN set -x && \ --yes \ "${PYTHON_SPECIFIER}" \ 'mamba' \ - 'jupyter_core' \ + 'jupyter_core' && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 07e5c53c7..d5c3ebe61 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -93,7 +93,7 @@ RUN set -x && \ --yes \ "${PYTHON_SPECIFIER}" \ 'mamba' \ - 'jupyter_core' \ + 'jupyter_core' && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ From 3b41c2645b14e2ab8d4c4fc02df42bb319fc5333 Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 16 Oct 2023 19:27:56 -0400 Subject: [PATCH 22/36] feat(docker): enable buildkit --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bb5fa1b2e..dcc46d43a 100644 --- a/Makefile +++ b/Makefile @@ -206,7 +206,7 @@ build/%: ## build the latest image # End repo with exactly one trailing slash, unless it is empty REPO=$$(echo "$(REPO)" | sed 's:/*$$:/:' | sed 's:^\s*/*\s*$$::') &&\ IMAGE_NAME="$${REPO}$(notdir $@):$(TAG)" && \ - DOCKER_BUILDKIT=0 docker build $(DARGS) --rm --force-rm -t $$IMAGE_NAME ./output/$(notdir $@) && \ + docker build $(DARGS) --rm --force-rm -t $$IMAGE_NAME ./output/$(notdir $@) && \ echo -n "Built image $$IMAGE_NAME of size: " && \ docker images $$IMAGE_NAME --format "{{.Size}}" && \ echo "full_image_name=$$IMAGE_NAME" >> $(GITHUB_OUTPUT) && \ From dadcc078e78dd2ed920ae345b7e87625a6d44038 Mon Sep 17 00:00:00 2001 From: souheil Date: Tue, 17 Oct 2023 00:51:51 -0400 Subject: [PATCH 23/36] fix(odbcdrives): remove alien and unixodbc (available upstream) --- docker-bits/5_DB-Drivers.Dockerfile | 4 ++-- output/jupyterlab-cpu/Dockerfile | 4 ++-- output/jupyterlab-pytorch/Dockerfile | 4 ++-- output/jupyterlab-tensorflow/Dockerfile | 4 ++-- output/rstudio/Dockerfile | 4 ++-- output/sas/Dockerfile | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docker-bits/5_DB-Drivers.Dockerfile b/docker-bits/5_DB-Drivers.Dockerfile index 4a73cb4da..96e55819e 100644 --- a/docker-bits/5_DB-Drivers.Dockerfile +++ b/docker-bits/5_DB-Drivers.Dockerfile @@ -1,7 +1,7 @@ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 39c435313..15ea3b758 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -135,8 +135,8 @@ RUN \ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index a63a6556c..3a4153bd2 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -157,8 +157,8 @@ RUN \ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 1c6ed8a3b..97ab8d318 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -263,8 +263,8 @@ RUN \ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/rstudio/Dockerfile b/output/rstudio/Dockerfile index 945829497..e7ca41980 100644 --- a/output/rstudio/Dockerfile +++ b/output/rstudio/Dockerfile @@ -135,8 +135,8 @@ RUN \ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 613ef9eed..043098b0e 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -137,8 +137,8 @@ RUN \ # ODBC drivers RUN apt-get update && \ - apt-get install -y alien unixodbc unixodbc-dev && \ - pip install --no-cache-dir --quiet 'pyodbc==4.0.35' && \ + apt-get install -y unixodbc-dev && \ + pip install --no-cache-dir --quiet pyodbc && \ rm -rf /var/lib/apt/lists/* && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER From aef66ece0d494e46c33464eea8d2d16b76e6304a Mon Sep 17 00:00:00 2001 From: souheil Date: Tue, 17 Oct 2023 10:36:18 -0400 Subject: [PATCH 24/36] refactor(jupylab): concat pip/mamba install --- docker-bits/3_Kubeflow.Dockerfile | 4 +-- docker-bits/6_jupyterlab.Dockerfile | 31 +++++++++------------- output/jupyterlab-cpu/Dockerfile | 35 ++++++++++--------------- output/jupyterlab-pytorch/Dockerfile | 35 ++++++++++--------------- output/jupyterlab-tensorflow/Dockerfile | 35 ++++++++++--------------- output/remote-desktop/Dockerfile | 4 +-- output/rstudio/Dockerfile | 4 +-- output/sas/Dockerfile | 35 ++++++++++--------------- 8 files changed, 74 insertions(+), 109 deletions(-) diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index f37dd6e7a..6c8f1f9fd 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -3,8 +3,8 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ - 'Pillow==9.4.0' \ - 'PyYAML==6.0.1' \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ diff --git a/docker-bits/6_jupyterlab.Dockerfile b/docker-bits/6_jupyterlab.Dockerfile index a981b6389..381d46624 100644 --- a/docker-bits/6_jupyterlab.Dockerfile +++ b/docker-bits/6_jupyterlab.Dockerfile @@ -27,9 +27,10 @@ RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \ && dpkg -i ./vscode.deb \ && rm ./vscode.deb \ && rm -f /etc/apt/sources.list.d/vscode.list \ - && mkdir -p $CS_TEMP_HOME/Machine - -RUN code-server --install-extension ms-python.python@2023.12.0 && \ + && mkdir -p $CS_TEMP_HOME/Machine \ + && \ + # Manage extensions + code-server --install-extension ms-python.python@2023.12.0 && \ code-server --install-extension REditorSupport.r@2.8.1 && \ code-server --install-extension ms-ceintl.vscode-language-pack-fr@1.79.0 && \ code-server --install-extension quarto.quarto@1.90.1 && \ @@ -48,16 +49,12 @@ COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json COPY languagepacks.json $CS_TEMP_HOME/ RUN pip install \ - 'git+https://github.com/betatim/vscode-binder' \ + 'git+https://github.com/betatim/vscode-binder' && \ # jupyter_contrib_nbextensions likes to be installed with pip - 'jupyter_contrib_nbextensions' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER -# Default environment -RUN mamba install --quiet --yes -c plotly -c conda-forge \ + mamba install --quiet --yes -c plotly -c conda-forge \ + 'jupyter_contrib_nbextensions' \ 'jupyter-dash' \ - 'pillow' \ - 'pyyaml' \ + 'plotly' \ 'ipywidgets' \ 'markupsafe' \ 'ipympl' \ @@ -67,21 +64,16 @@ RUN mamba install --quiet --yes -c plotly -c conda-forge \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN mamba clean --all -f -y && \ - jupyter serverextension enable --py jupyter_server_proxy && \ + mamba clean --all -f -y && \ + jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension install \ '@jupyterlab/translation-extension' \ '@jupyterlab/server-proxy@2.1.2' \ - 'jupyterlab-plotly@4.14.3' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ jupyter lab clean && \ - npm cache clean --force && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -93,7 +85,7 @@ RUN mamba clean --all -f -y && \ # Install python, R, Julia and other useful language servers RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ /opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \ - conda install -c conda-forge \ + mamba install -c conda-forge \ 'r-languageserver' \ 'python-lsp-server' \ && \ @@ -104,6 +96,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'unified-language-server' \ 'yaml-language-server@0.18.0' && \ mamba clean --all -f -y && \ + npm cache clean --force && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 15ea3b758..a3619a4e6 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -34,8 +34,8 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ - 'Pillow==9.4.0' \ - 'PyYAML==6.0.1' \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ @@ -174,9 +174,10 @@ RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \ && dpkg -i ./vscode.deb \ && rm ./vscode.deb \ && rm -f /etc/apt/sources.list.d/vscode.list \ - && mkdir -p $CS_TEMP_HOME/Machine - -RUN code-server --install-extension ms-python.python@2023.12.0 && \ + && mkdir -p $CS_TEMP_HOME/Machine \ + && \ + # Manage extensions + code-server --install-extension ms-python.python@2023.12.0 && \ code-server --install-extension REditorSupport.r@2.8.1 && \ code-server --install-extension ms-ceintl.vscode-language-pack-fr@1.79.0 && \ code-server --install-extension quarto.quarto@1.90.1 && \ @@ -195,16 +196,12 @@ COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json COPY languagepacks.json $CS_TEMP_HOME/ RUN pip install \ - 'git+https://github.com/betatim/vscode-binder' \ + 'git+https://github.com/betatim/vscode-binder' && \ # jupyter_contrib_nbextensions likes to be installed with pip - 'jupyter_contrib_nbextensions' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER -# Default environment -RUN mamba install --quiet --yes -c plotly -c conda-forge \ + mamba install --quiet --yes -c plotly -c conda-forge \ + 'jupyter_contrib_nbextensions' \ 'jupyter-dash' \ - 'pillow' \ - 'pyyaml' \ + 'plotly' \ 'ipywidgets' \ 'markupsafe' \ 'ipympl' \ @@ -214,21 +211,16 @@ RUN mamba install --quiet --yes -c plotly -c conda-forge \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN mamba clean --all -f -y && \ - jupyter serverextension enable --py jupyter_server_proxy && \ + mamba clean --all -f -y && \ + jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension install \ '@jupyterlab/translation-extension' \ '@jupyterlab/server-proxy@2.1.2' \ - 'jupyterlab-plotly@4.14.3' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ jupyter lab clean && \ - npm cache clean --force && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -240,7 +232,7 @@ RUN mamba clean --all -f -y && \ # Install python, R, Julia and other useful language servers RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ /opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \ - conda install -c conda-forge \ + mamba install -c conda-forge \ 'r-languageserver' \ 'python-lsp-server' \ && \ @@ -251,6 +243,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'unified-language-server' \ 'yaml-language-server@0.18.0' && \ mamba clean --all -f -y && \ + npm cache clean --force && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index 3a4153bd2..abebd924b 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -56,8 +56,8 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ - 'Pillow==9.4.0' \ - 'PyYAML==6.0.1' \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ @@ -196,9 +196,10 @@ RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \ && dpkg -i ./vscode.deb \ && rm ./vscode.deb \ && rm -f /etc/apt/sources.list.d/vscode.list \ - && mkdir -p $CS_TEMP_HOME/Machine - -RUN code-server --install-extension ms-python.python@2023.12.0 && \ + && mkdir -p $CS_TEMP_HOME/Machine \ + && \ + # Manage extensions + code-server --install-extension ms-python.python@2023.12.0 && \ code-server --install-extension REditorSupport.r@2.8.1 && \ code-server --install-extension ms-ceintl.vscode-language-pack-fr@1.79.0 && \ code-server --install-extension quarto.quarto@1.90.1 && \ @@ -217,16 +218,12 @@ COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json COPY languagepacks.json $CS_TEMP_HOME/ RUN pip install \ - 'git+https://github.com/betatim/vscode-binder' \ + 'git+https://github.com/betatim/vscode-binder' && \ # jupyter_contrib_nbextensions likes to be installed with pip - 'jupyter_contrib_nbextensions' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER -# Default environment -RUN mamba install --quiet --yes -c plotly -c conda-forge \ + mamba install --quiet --yes -c plotly -c conda-forge \ + 'jupyter_contrib_nbextensions' \ 'jupyter-dash' \ - 'pillow' \ - 'pyyaml' \ + 'plotly' \ 'ipywidgets' \ 'markupsafe' \ 'ipympl' \ @@ -236,21 +233,16 @@ RUN mamba install --quiet --yes -c plotly -c conda-forge \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN mamba clean --all -f -y && \ - jupyter serverextension enable --py jupyter_server_proxy && \ + mamba clean --all -f -y && \ + jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension install \ '@jupyterlab/translation-extension' \ '@jupyterlab/server-proxy@2.1.2' \ - 'jupyterlab-plotly@4.14.3' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ jupyter lab clean && \ - npm cache clean --force && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -262,7 +254,7 @@ RUN mamba clean --all -f -y && \ # Install python, R, Julia and other useful language servers RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ /opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \ - conda install -c conda-forge \ + mamba install -c conda-forge \ 'r-languageserver' \ 'python-lsp-server' \ && \ @@ -273,6 +265,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'unified-language-server' \ 'yaml-language-server@0.18.0' && \ mamba clean --all -f -y && \ + npm cache clean --force && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 97ab8d318..2f584b332 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -162,8 +162,8 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ - 'Pillow==9.4.0' \ - 'PyYAML==6.0.1' \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ @@ -302,9 +302,10 @@ RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \ && dpkg -i ./vscode.deb \ && rm ./vscode.deb \ && rm -f /etc/apt/sources.list.d/vscode.list \ - && mkdir -p $CS_TEMP_HOME/Machine - -RUN code-server --install-extension ms-python.python@2023.12.0 && \ + && mkdir -p $CS_TEMP_HOME/Machine \ + && \ + # Manage extensions + code-server --install-extension ms-python.python@2023.12.0 && \ code-server --install-extension REditorSupport.r@2.8.1 && \ code-server --install-extension ms-ceintl.vscode-language-pack-fr@1.79.0 && \ code-server --install-extension quarto.quarto@1.90.1 && \ @@ -323,16 +324,12 @@ COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json COPY languagepacks.json $CS_TEMP_HOME/ RUN pip install \ - 'git+https://github.com/betatim/vscode-binder' \ + 'git+https://github.com/betatim/vscode-binder' && \ # jupyter_contrib_nbextensions likes to be installed with pip - 'jupyter_contrib_nbextensions' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER -# Default environment -RUN mamba install --quiet --yes -c plotly -c conda-forge \ + mamba install --quiet --yes -c plotly -c conda-forge \ + 'jupyter_contrib_nbextensions' \ 'jupyter-dash' \ - 'pillow' \ - 'pyyaml' \ + 'plotly' \ 'ipywidgets' \ 'markupsafe' \ 'ipympl' \ @@ -342,21 +339,16 @@ RUN mamba install --quiet --yes -c plotly -c conda-forge \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN mamba clean --all -f -y && \ - jupyter serverextension enable --py jupyter_server_proxy && \ + mamba clean --all -f -y && \ + jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension install \ '@jupyterlab/translation-extension' \ '@jupyterlab/server-proxy@2.1.2' \ - 'jupyterlab-plotly@4.14.3' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ jupyter lab clean && \ - npm cache clean --force && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -368,7 +360,7 @@ RUN mamba clean --all -f -y && \ # Install python, R, Julia and other useful language servers RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ /opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \ - conda install -c conda-forge \ + mamba install -c conda-forge \ 'r-languageserver' \ 'python-lsp-server' \ && \ @@ -379,6 +371,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'unified-language-server' \ 'yaml-language-server@0.18.0' && \ mamba clean --all -f -y && \ + npm cache clean --force && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index d5c3ebe61..076dd719a 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -110,8 +110,8 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ - 'Pillow==9.4.0' \ - 'PyYAML==6.0.1' \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ diff --git a/output/rstudio/Dockerfile b/output/rstudio/Dockerfile index e7ca41980..60bc7e532 100644 --- a/output/rstudio/Dockerfile +++ b/output/rstudio/Dockerfile @@ -34,8 +34,8 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ - 'Pillow==9.4.0' \ - 'PyYAML==6.0.1' \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 043098b0e..8cc581ad5 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -36,8 +36,8 @@ COPY aaw-suspend-server.sh /usr/local/bin # https://github.com/StatCan/aaw-kubeflow-containers/issues/293 RUN mamba install --quiet \ - 'Pillow==9.4.0' \ - 'PyYAML==6.0.1' \ + 'pillow' \ + 'pyyaml' \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ @@ -176,9 +176,10 @@ RUN wget -q "${VSCODE_URL}" -O ./vscode.deb \ && dpkg -i ./vscode.deb \ && rm ./vscode.deb \ && rm -f /etc/apt/sources.list.d/vscode.list \ - && mkdir -p $CS_TEMP_HOME/Machine - -RUN code-server --install-extension ms-python.python@2023.12.0 && \ + && mkdir -p $CS_TEMP_HOME/Machine \ + && \ + # Manage extensions + code-server --install-extension ms-python.python@2023.12.0 && \ code-server --install-extension REditorSupport.r@2.8.1 && \ code-server --install-extension ms-ceintl.vscode-language-pack-fr@1.79.0 && \ code-server --install-extension quarto.quarto@1.90.1 && \ @@ -197,16 +198,12 @@ COPY vscode-overrides.json $CS_TEMP_HOME/Machine/settings.json COPY languagepacks.json $CS_TEMP_HOME/ RUN pip install \ - 'git+https://github.com/betatim/vscode-binder' \ + 'git+https://github.com/betatim/vscode-binder' && \ # jupyter_contrib_nbextensions likes to be installed with pip - 'jupyter_contrib_nbextensions' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER -# Default environment -RUN mamba install --quiet --yes -c plotly -c conda-forge \ + mamba install --quiet --yes -c plotly -c conda-forge \ + 'jupyter_contrib_nbextensions' \ 'jupyter-dash' \ - 'pillow' \ - 'pyyaml' \ + 'plotly' \ 'ipywidgets' \ 'markupsafe' \ 'ipympl' \ @@ -216,21 +213,16 @@ RUN mamba install --quiet --yes -c plotly -c conda-forge \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -RUN mamba clean --all -f -y && \ - jupyter serverextension enable --py jupyter_server_proxy && \ + mamba clean --all -f -y && \ + jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension install \ '@jupyterlab/translation-extension' \ '@jupyterlab/server-proxy@2.1.2' \ - 'jupyterlab-plotly@4.14.3' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ jupyter lab clean && \ - npm cache clean --force && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -242,7 +234,7 @@ RUN mamba clean --all -f -y && \ # Install python, R, Julia and other useful language servers RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ /opt/conda/bin/R --silent --slave --no-save --no-restore -e 'install.packages("languageserver", repos="https://cran.r-project.org/")' && \ - conda install -c conda-forge \ + mamba install -c conda-forge \ 'r-languageserver' \ 'python-lsp-server' \ && \ @@ -253,6 +245,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'unified-language-server' \ 'yaml-language-server@0.18.0' && \ mamba clean --all -f -y && \ + npm cache clean --force && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER From 1347d2cd1e3ed07dfd0ed216ae20650b3471acde Mon Sep 17 00:00:00 2001 From: souheil Date: Tue, 17 Oct 2023 10:59:03 -0400 Subject: [PATCH 25/36] refactor(jupylab): update pinned extension --- docker-bits/6_jupyterlab.Dockerfile | 2 +- output/jupyterlab-cpu/Dockerfile | 2 +- output/jupyterlab-pytorch/Dockerfile | 2 +- output/jupyterlab-tensorflow/Dockerfile | 2 +- output/sas/Dockerfile | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-bits/6_jupyterlab.Dockerfile b/docker-bits/6_jupyterlab.Dockerfile index 381d46624..0602d3495 100644 --- a/docker-bits/6_jupyterlab.Dockerfile +++ b/docker-bits/6_jupyterlab.Dockerfile @@ -69,7 +69,7 @@ RUN pip install \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension install \ '@jupyterlab/translation-extension' \ - '@jupyterlab/server-proxy@2.1.2' \ + '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index a3619a4e6..ca0b3aed4 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -216,7 +216,7 @@ RUN pip install \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension install \ '@jupyterlab/translation-extension' \ - '@jupyterlab/server-proxy@2.1.2' \ + '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index abebd924b..da9f1a3fa 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -238,7 +238,7 @@ RUN pip install \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension install \ '@jupyterlab/translation-extension' \ - '@jupyterlab/server-proxy@2.1.2' \ + '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 2f584b332..4b6c92bff 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -344,7 +344,7 @@ RUN pip install \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension install \ '@jupyterlab/translation-extension' \ - '@jupyterlab/server-proxy@2.1.2' \ + '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 8cc581ad5..0be099b7b 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -218,7 +218,7 @@ RUN pip install \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension install \ '@jupyterlab/translation-extension' \ - '@jupyterlab/server-proxy@2.1.2' \ + '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ && \ jupyter lab build && \ From b251e023f433b1c514151d583d4aae638109d610 Mon Sep 17 00:00:00 2001 From: souheil Date: Wed, 18 Oct 2023 14:17:33 -0400 Subject: [PATCH 26/36] fix(jupyterlab): labextension deprecated --- docker-bits/6_jupyterlab.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-bits/6_jupyterlab.Dockerfile b/docker-bits/6_jupyterlab.Dockerfile index 0602d3495..9604afbcf 100644 --- a/docker-bits/6_jupyterlab.Dockerfile +++ b/docker-bits/6_jupyterlab.Dockerfile @@ -67,7 +67,7 @@ RUN pip install \ mamba clean --all -f -y && \ jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ - jupyter labextension install \ + jupyter labextension enable \ '@jupyterlab/translation-extension' \ '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ From 5efea3a15b33f10004a5e499df1d0853a82af684 Mon Sep 17 00:00:00 2001 From: souheil Date: Wed, 18 Oct 2023 14:27:15 -0400 Subject: [PATCH 27/36] refactor(clean-layer): restructure script call --- docker-bits/0_Rocker.Dockerfile | 7 +++++-- docker-bits/0_cpu.Dockerfile | 5 ++++- docker-bits/0_cpu_sas.Dockerfile | 7 +++++-- docker-bits/6_remote-desktop.Dockerfile | 3 --- .../docker-stacks-datascience-notebook/clean-layer.sh | 5 ++--- output/jupyterlab-cpu/Dockerfile | 7 +++++-- output/jupyterlab-cpu/clean-layer.sh | 5 ++--- output/jupyterlab-pytorch/Dockerfile | 7 +++++-- output/jupyterlab-pytorch/clean-layer.sh | 5 ++--- output/jupyterlab-tensorflow/Dockerfile | 7 +++++-- output/jupyterlab-tensorflow/clean-layer.sh | 5 ++--- output/remote-desktop/Dockerfile | 10 +++++----- output/remote-desktop/clean-layer.sh | 5 ++--- output/rstudio/Dockerfile | 5 ++++- output/rstudio/clean-layer.sh | 5 ++--- output/sas/Dockerfile | 7 +++++-- output/sas/clean-layer.sh | 5 ++--- resources/common/clean-layer.sh | 5 ++--- 18 files changed, 59 insertions(+), 46 deletions(-) diff --git a/docker-bits/0_Rocker.Dockerfile b/docker-bits/0_Rocker.Dockerfile index 1ba5becaa..e95d282fb 100644 --- a/docker-bits/0_Rocker.Dockerfile +++ b/docker-bits/0_Rocker.Dockerfile @@ -22,7 +22,11 @@ ENV PATH="/home/jovyan/.local/bin/:${PATH}" #Fix-permissions COPY remote-desktop/fix-permissions /usr/bin/fix-permissions -RUN chmod u+x /usr/bin/fix-permissions +#clean up +COPY clean-layer.sh /usr/bin/clean-layer.sh + +RUN chmod u+x /usr/bin/fix-permissions \ + && chmod +x /usr/bin/clean-layer.sh RUN apt-get update --yes \ && apt-get install --yes python3-pip tini language-pack-fr \ @@ -36,7 +40,6 @@ RUN /rocker_scripts/install_shiny_server.sh \ RUN chmod 555 /usr/local/lib/R /usr/local/lib/R/site-library/ - # ARG CONDA_VERSION=py38_4.10.3 # ARG CONDA_MD5=14da4a9a44b337f7ccb8363537f65b9c ARG PYTHON_VERSION=3.11 diff --git a/docker-bits/0_cpu.Dockerfile b/docker-bits/0_cpu.Dockerfile index 752d89dcc..78efa1545 100644 --- a/docker-bits/0_cpu.Dockerfile +++ b/docker-bits/0_cpu.Dockerfile @@ -10,7 +10,10 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ && apt-get upgrade --yes libwebp7 \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh \ No newline at end of file diff --git a/docker-bits/0_cpu_sas.Dockerfile b/docker-bits/0_cpu_sas.Dockerfile index f96c29355..85f75878b 100644 --- a/docker-bits/0_cpu_sas.Dockerfile +++ b/docker-bits/0_cpu_sas.Dockerfile @@ -12,12 +12,15 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh #updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 #TODO: Evaluate if this is still necessary when updating the base image RUN pip install --force-reinstall cryptography==39.0.1 && \ fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + fix-permissions /home/$NB_USER \ No newline at end of file diff --git a/docker-bits/6_remote-desktop.Dockerfile b/docker-bits/6_remote-desktop.Dockerfile index ad5e9d9ca..ecf0150b3 100644 --- a/docker-bits/6_remote-desktop.Dockerfile +++ b/docker-bits/6_remote-desktop.Dockerfile @@ -6,9 +6,6 @@ ENV XDG_DATA_HOME=/etc/share ENV VSCODE_DIR=$XDG_DATA_HOME/code ENV VSCODE_EXTENSIONS=$VSCODE_DIR/extensions -COPY clean-layer.sh /usr/bin/clean-layer.sh -RUN chmod +x /usr/bin/clean-layer.sh - ENV DEBIAN_FRONTEND noninteractive RUN apt-get -y update \ && apt-get install -y dbus-x11 \ diff --git a/output/docker-stacks-datascience-notebook/clean-layer.sh b/output/docker-stacks-datascience-notebook/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/docker-stacks-datascience-notebook/clean-layer.sh +++ b/output/docker-stacks-datascience-notebook/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index ca0b3aed4..79a8ef90f 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -20,10 +20,13 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ && apt-get upgrade --yes libwebp7 \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh ############################### ### docker-bits/3_Kubeflow.Dockerfile @@ -214,7 +217,7 @@ RUN pip install \ mamba clean --all -f -y && \ jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ - jupyter labextension install \ + jupyter labextension enable \ '@jupyterlab/translation-extension' \ '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ diff --git a/output/jupyterlab-cpu/clean-layer.sh b/output/jupyterlab-cpu/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/jupyterlab-cpu/clean-layer.sh +++ b/output/jupyterlab-cpu/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index da9f1a3fa..2289e1460 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -20,10 +20,13 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ && apt-get upgrade --yes libwebp7 \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh ############################### ### docker-bits/2_pytorch.Dockerfile @@ -236,7 +239,7 @@ RUN pip install \ mamba clean --all -f -y && \ jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ - jupyter labextension install \ + jupyter labextension enable \ '@jupyterlab/translation-extension' \ '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ diff --git a/output/jupyterlab-pytorch/clean-layer.sh b/output/jupyterlab-pytorch/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/jupyterlab-pytorch/clean-layer.sh +++ b/output/jupyterlab-pytorch/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 4b6c92bff..6f0caef6b 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -20,10 +20,13 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ && apt-get upgrade --yes libwebp7 \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh ############################### ### docker-bits/1_CUDA-11.8.0.Dockerfile @@ -342,7 +345,7 @@ RUN pip install \ mamba clean --all -f -y && \ jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ - jupyter labextension install \ + jupyter labextension enable \ '@jupyterlab/translation-extension' \ '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ diff --git a/output/jupyterlab-tensorflow/clean-layer.sh b/output/jupyterlab-tensorflow/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/jupyterlab-tensorflow/clean-layer.sh +++ b/output/jupyterlab-tensorflow/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 076dd719a..72b2272bd 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -27,7 +27,11 @@ ENV PATH="/home/jovyan/.local/bin/:${PATH}" #Fix-permissions COPY remote-desktop/fix-permissions /usr/bin/fix-permissions -RUN chmod u+x /usr/bin/fix-permissions +#clean up +COPY clean-layer.sh /usr/bin/clean-layer.sh + +RUN chmod u+x /usr/bin/fix-permissions \ + && chmod +x /usr/bin/clean-layer.sh RUN apt-get update --yes \ && apt-get install --yes python3-pip tini language-pack-fr \ @@ -41,7 +45,6 @@ RUN /rocker_scripts/install_shiny_server.sh \ RUN chmod 555 /usr/local/lib/R /usr/local/lib/R/site-library/ - # ARG CONDA_VERSION=py38_4.10.3 # ARG CONDA_MD5=14da4a9a44b337f7ccb8363537f65b9c ARG PYTHON_VERSION=3.11 @@ -217,9 +220,6 @@ ENV XDG_DATA_HOME=/etc/share ENV VSCODE_DIR=$XDG_DATA_HOME/code ENV VSCODE_EXTENSIONS=$VSCODE_DIR/extensions -COPY clean-layer.sh /usr/bin/clean-layer.sh -RUN chmod +x /usr/bin/clean-layer.sh - ENV DEBIAN_FRONTEND noninteractive RUN apt-get -y update \ && apt-get install -y dbus-x11 \ diff --git a/output/remote-desktop/clean-layer.sh b/output/remote-desktop/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/remote-desktop/clean-layer.sh +++ b/output/remote-desktop/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/rstudio/Dockerfile b/output/rstudio/Dockerfile index 60bc7e532..91197ee73 100644 --- a/output/rstudio/Dockerfile +++ b/output/rstudio/Dockerfile @@ -20,10 +20,13 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ && apt-get upgrade --yes libwebp7 \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh ############################### ### docker-bits/3_Kubeflow.Dockerfile diff --git a/output/rstudio/clean-layer.sh b/output/rstudio/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/rstudio/clean-layer.sh +++ b/output/rstudio/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 0be099b7b..8181dd93f 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -17,9 +17,12 @@ USER root ENV PATH="/home/jovyan/.local/bin/:${PATH}" +COPY clean-layer.sh /usr/bin/clean-layer.sh + RUN apt-get update --yes \ && apt-get install --yes language-pack-fr \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && chmod +x /usr/bin/clean-layer.sh #updates package to fix CVE-2023-0286 https://github.com/StatCan/aaw-private/issues/57 #TODO: Evaluate if this is still necessary when updating the base image @@ -216,7 +219,7 @@ RUN pip install \ mamba clean --all -f -y && \ jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ - jupyter labextension install \ + jupyter labextension enable \ '@jupyterlab/translation-extension' \ '@jupyterlab/server-proxy' \ 'nbdime-jupyterlab' \ diff --git a/output/sas/clean-layer.sh b/output/sas/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/output/sas/clean-layer.sh +++ b/output/sas/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete diff --git a/resources/common/clean-layer.sh b/resources/common/clean-layer.sh index a51f4af3e..ae31dc3a3 100755 --- a/resources/common/clean-layer.sh +++ b/resources/common/clean-layer.sh @@ -23,11 +23,10 @@ rm -rf /tmp/* /var/tmp/* $HOME/.cache/* /var/cache/apt/* rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/* # Clean conda -if [ -x "$(command -v conda)" ]; then +if [ -x "$(command -v mamba)" ]; then # Full Conda Cleanup - conda clean --all -f -y + mamba clean --all -f -y # Remove source cache files - conda build purge-all if [ -d $CONDA_DIR ]; then # Cleanup python bytecode files - not needed: https://jcrist.github.io/conda-docker-tips.html find $CONDA_DIR -type f -name '*.pyc' -delete From afcbba9962dde6d73778b3c88a753f4d0aca748f Mon Sep 17 00:00:00 2001 From: souheil Date: Wed, 18 Oct 2023 16:40:01 -0400 Subject: [PATCH 28/36] fix(makefile): re-disable buildkit...? --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index dcc46d43a..bb5fa1b2e 100644 --- a/Makefile +++ b/Makefile @@ -206,7 +206,7 @@ build/%: ## build the latest image # End repo with exactly one trailing slash, unless it is empty REPO=$$(echo "$(REPO)" | sed 's:/*$$:/:' | sed 's:^\s*/*\s*$$::') &&\ IMAGE_NAME="$${REPO}$(notdir $@):$(TAG)" && \ - docker build $(DARGS) --rm --force-rm -t $$IMAGE_NAME ./output/$(notdir $@) && \ + DOCKER_BUILDKIT=0 docker build $(DARGS) --rm --force-rm -t $$IMAGE_NAME ./output/$(notdir $@) && \ echo -n "Built image $$IMAGE_NAME of size: " && \ docker images $$IMAGE_NAME --format "{{.Size}}" && \ echo "full_image_name=$$IMAGE_NAME" >> $(GITHUB_OUTPUT) && \ From 596e4c14a165d2a97d786e1e8baf3deba91a80c8 Mon Sep 17 00:00:00 2001 From: souheil-yazji Date: Thu, 19 Oct 2023 19:19:54 +0000 Subject: [PATCH 29/36] refactor(layer-cache): add clean-layer.sh to all conda/mamba/npm runs --- docker-bits/0_Rocker.Dockerfile | 2 +- docker-bits/2_cpu.Dockerfile | 2 +- docker-bits/2_pytorch.Dockerfile | 2 +- docker-bits/2_tensorflow.Dockerfile | 1 + docker-bits/3_Kubeflow.Dockerfile | 1 + docker-bits/6_jupyterlab.Dockerfile | 5 ++--- docker-bits/6_rstudio.Dockerfile | 4 ++-- output/jupyterlab-cpu/Dockerfile | 6 +++--- output/jupyterlab-pytorch/Dockerfile | 8 ++++---- output/jupyterlab-tensorflow/Dockerfile | 7 ++++--- output/remote-desktop/Dockerfile | 3 ++- output/rstudio/Dockerfile | 5 +++-- output/sas/Dockerfile | 10 +++++----- 13 files changed, 30 insertions(+), 26 deletions(-) diff --git a/docker-bits/0_Rocker.Dockerfile b/docker-bits/0_Rocker.Dockerfile index e95d282fb..9d03a02ab 100644 --- a/docker-bits/0_Rocker.Dockerfile +++ b/docker-bits/0_Rocker.Dockerfile @@ -95,6 +95,6 @@ RUN set -x && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ - mamba clean --all -f -y && \ + clean-layer.sh && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" \ No newline at end of file diff --git a/docker-bits/2_cpu.Dockerfile b/docker-bits/2_cpu.Dockerfile index 98581e0e4..4bfa9d62b 100644 --- a/docker-bits/2_cpu.Dockerfile +++ b/docker-bits/2_cpu.Dockerfile @@ -12,6 +12,6 @@ RUN mamba create -n pycpu --yes \ gensim==4.3.0 nltk==3.8.1 spacy==3.5.0 \ pytorch==1.13.1 torchaudio==0.13.1 torchvision==0.14.1 cpuonly==2.0 \ -c pytorch -c conda-forge && \ - mamba clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/docker-bits/2_pytorch.Dockerfile b/docker-bits/2_pytorch.Dockerfile index 5b73592bf..ab2bdabf2 100644 --- a/docker-bits/2_pytorch.Dockerfile +++ b/docker-bits/2_pytorch.Dockerfile @@ -10,7 +10,7 @@ RUN mamba create -n torch && \ gputil \ # pytorch-cuda are the nvidia cuda drivers pytorch-cuda=11.8 && \ - mamba clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ python -m ipykernel install --user --name torch --display-name "PyTorch" diff --git a/docker-bits/2_tensorflow.Dockerfile b/docker-bits/2_tensorflow.Dockerfile index 4ac789207..a4b12df0e 100644 --- a/docker-bits/2_tensorflow.Dockerfile +++ b/docker-bits/2_tensorflow.Dockerfile @@ -4,5 +4,6 @@ RUN mamba install --quiet --yes \ keras \ ipykernel \ && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index 6c8f1f9fd..08aa2c549 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -10,6 +10,7 @@ RUN mamba install --quiet \ 's3fs' \ 'fire==0.5.0' && \ pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh diff --git a/docker-bits/6_jupyterlab.Dockerfile b/docker-bits/6_jupyterlab.Dockerfile index 9604afbcf..704fa6917 100644 --- a/docker-bits/6_jupyterlab.Dockerfile +++ b/docker-bits/6_jupyterlab.Dockerfile @@ -64,7 +64,6 @@ RUN pip install \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - mamba clean --all -f -y && \ jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension enable \ @@ -74,6 +73,7 @@ RUN pip install \ && \ jupyter lab build && \ jupyter lab clean && \ + clean-layer.sh && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -95,8 +95,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server@0.18.0' && \ - mamba clean --all -f -y && \ - npm cache clean --force && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/docker-bits/6_rstudio.Dockerfile b/docker-bits/6_rstudio.Dockerfile index 365964568..c84bf18c7 100644 --- a/docker-bits/6_rstudio.Dockerfile +++ b/docker-bits/6_rstudio.Dockerfile @@ -7,7 +7,7 @@ RUN mkdir -p /etc/rstudio && \ ENV PATH=$PATH:/usr/lib/rstudio-server/bin # Install some default R packages -RUN conda install --quiet --yes \ +RUN mamba install --quiet --yes \ 'r-rodbc==1.3_20' \ 'r-tidyverse==1.3.2' \ 'r-arrow==12.0.0' \ @@ -19,7 +19,7 @@ RUN conda install --quiet --yes \ 'r-e1071==1.7_13' \ 'r-markdown==1.7' \ && \ - conda clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 79a8ef90f..40f6daa59 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -44,6 +44,7 @@ RUN mamba install --quiet \ 's3fs' \ 'fire==0.5.0' && \ pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh @@ -214,7 +215,6 @@ RUN pip install \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - mamba clean --all -f -y && \ jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension enable \ @@ -224,6 +224,7 @@ RUN pip install \ && \ jupyter lab build && \ jupyter lab clean && \ + clean-layer.sh && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -245,8 +246,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server@0.18.0' && \ - mamba clean --all -f -y && \ - npm cache clean --force && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index 2289e1460..cf30dad5c 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -44,7 +44,7 @@ RUN mamba create -n torch && \ gputil \ # pytorch-cuda are the nvidia cuda drivers pytorch-cuda=11.8 && \ - mamba clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ python -m ipykernel install --user --name torch --display-name "PyTorch" @@ -66,6 +66,7 @@ RUN mamba install --quiet \ 's3fs' \ 'fire==0.5.0' && \ pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh @@ -236,7 +237,6 @@ RUN pip install \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - mamba clean --all -f -y && \ jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension enable \ @@ -246,6 +246,7 @@ RUN pip install \ && \ jupyter lab build && \ jupyter lab clean && \ + clean-layer.sh && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -267,8 +268,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server@0.18.0' && \ - mamba clean --all -f -y && \ - npm cache clean --force && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 6f0caef6b..511f15a00 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -153,6 +153,7 @@ RUN mamba install --quiet --yes \ keras \ ipykernel \ && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER @@ -172,6 +173,7 @@ RUN mamba install --quiet \ 's3fs' \ 'fire==0.5.0' && \ pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh @@ -342,7 +344,6 @@ RUN pip install \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - mamba clean --all -f -y && \ jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension enable \ @@ -352,6 +353,7 @@ RUN pip install \ && \ jupyter lab build && \ jupyter lab clean && \ + clean-layer.sh && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -373,8 +375,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server@0.18.0' && \ - mamba clean --all -f -y && \ - npm cache clean --force && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 72b2272bd..97c48e249 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -100,7 +100,7 @@ RUN set -x && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ - mamba clean --all -f -y && \ + clean-layer.sh && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" @@ -120,6 +120,7 @@ RUN mamba install --quiet \ 's3fs' \ 'fire==0.5.0' && \ pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh diff --git a/output/rstudio/Dockerfile b/output/rstudio/Dockerfile index 91197ee73..833a8ea6f 100644 --- a/output/rstudio/Dockerfile +++ b/output/rstudio/Dockerfile @@ -44,6 +44,7 @@ RUN mamba install --quiet \ 's3fs' \ 'fire==0.5.0' && \ pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh @@ -181,7 +182,7 @@ RUN mkdir -p /etc/rstudio && \ ENV PATH=$PATH:/usr/lib/rstudio-server/bin # Install some default R packages -RUN conda install --quiet --yes \ +RUN mamba install --quiet --yes \ 'r-rodbc==1.3_20' \ 'r-tidyverse==1.3.2' \ 'r-arrow==12.0.0' \ @@ -193,7 +194,7 @@ RUN conda install --quiet --yes \ 'r-e1071==1.7_13' \ 'r-markdown==1.7' \ && \ - conda clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 8181dd93f..979d1f639 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -46,6 +46,7 @@ RUN mamba install --quiet \ 's3fs' \ 'fire==0.5.0' && \ pip install 'kubeflow-training' && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER && \ chmod +x /usr/local/bin/aaw-suspend-server.sh @@ -216,7 +217,6 @@ RUN pip install \ 'nb_conda_kernels' \ 'jupyterlab-lsp' \ 'jupyter-lsp' && \ - mamba clean --all -f -y && \ jupyter server extension enable --py jupyter_server_proxy && \ jupyter nbextension enable codefolding/main --sys-prefix && \ jupyter labextension enable \ @@ -226,6 +226,7 @@ RUN pip install \ && \ jupyter lab build && \ jupyter lab clean && \ + clean-layer.sh && \ rm -rf /home/$NB_USER/.cache/yarn && \ rm -rf /home/$NB_USER/.node-gyp && \ fix-permissions $CONDA_DIR && \ @@ -247,8 +248,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server@0.18.0' && \ - mamba clean --all -f -y && \ - npm cache clean --force && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER @@ -325,7 +325,7 @@ RUN mkdir -p /etc/rstudio && \ ENV PATH=$PATH:/usr/lib/rstudio-server/bin # Install some default R packages -RUN conda install --quiet --yes \ +RUN mamba install --quiet --yes \ 'r-rodbc==1.3_20' \ 'r-tidyverse==1.3.2' \ 'r-arrow==12.0.0' \ @@ -337,7 +337,7 @@ RUN conda install --quiet --yes \ 'r-e1071==1.7_13' \ 'r-markdown==1.7' \ && \ - conda clean --all -f -y && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER From 2e53aaa27b3f3b9fa513b50b9c3bf72460eb46b1 Mon Sep 17 00:00:00 2001 From: souheil-yazji Date: Thu, 19 Oct 2023 19:53:52 +0000 Subject: [PATCH 30/36] fix(sqlite): update apt archive pre-install --- docker-bits/6_jupyterlab.Dockerfile | 3 ++- docker-bits/6_remote-desktop.Dockerfile | 3 ++- output/jupyterlab-cpu/Dockerfile | 3 ++- output/jupyterlab-pytorch/Dockerfile | 3 ++- output/jupyterlab-tensorflow/Dockerfile | 3 ++- output/remote-desktop/Dockerfile | 3 ++- output/sas/Dockerfile | 3 ++- 7 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docker-bits/6_jupyterlab.Dockerfile b/docker-bits/6_jupyterlab.Dockerfile index 704fa6917..6f205ed3d 100644 --- a/docker-bits/6_jupyterlab.Dockerfile +++ b/docker-bits/6_jupyterlab.Dockerfile @@ -109,7 +109,8 @@ ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_debian_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && mkdir -p ${OMPP_INSTALL_DIR} \ diff --git a/docker-bits/6_remote-desktop.Dockerfile b/docker-bits/6_remote-desktop.Dockerfile index ecf0150b3..c5161d293 100644 --- a/docker-bits/6_remote-desktop.Dockerfile +++ b/docker-bits/6_remote-desktop.Dockerfile @@ -281,7 +281,8 @@ ENV OMPP_GROUP=100 ENV OMPP_UID=$NB_UID ENV OMPP_GID=$NB_GID # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_ubuntu_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && tar -xf /tmp/ompp.tar.gz -C /tmp/ \ diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 40f6daa59..430516484 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -260,7 +260,8 @@ ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_debian_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && mkdir -p ${OMPP_INSTALL_DIR} \ diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index cf30dad5c..ffda8f01d 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -282,7 +282,8 @@ ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_debian_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && mkdir -p ${OMPP_INSTALL_DIR} \ diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 511f15a00..037524e3a 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -389,7 +389,8 @@ ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_debian_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && mkdir -p ${OMPP_INSTALL_DIR} \ diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index 97c48e249..d9f3ded7b 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -496,7 +496,8 @@ ENV OMPP_GROUP=100 ENV OMPP_UID=$NB_UID ENV OMPP_GID=$NB_GID # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_ubuntu_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && tar -xf /tmp/ompp.tar.gz -C /tmp/ \ diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 979d1f639..3cfab7788 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -262,7 +262,8 @@ ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} # OpenM++ expects sqlite to be installed (not just libsqlite) -RUN apt-get install --yes sqlite3 \ +RUN apt-get update --yes \ + && apt-get install --yes sqlite3 \ && wget -q https://github.com/openmpp/main/releases/download/v${OMPP_VERSION}/openmpp_debian_${OMPP_PKG_DATE}.tar.gz -O /tmp/ompp.tar.gz \ && echo "${SHA256ompp} /tmp/ompp.tar.gz" | sha256sum -c - \ && mkdir -p ${OMPP_INSTALL_DIR} \ From 689e1981a1b238da0d802e504b9dd9b9695167a9 Mon Sep 17 00:00:00 2001 From: souheil Date: Sun, 22 Oct 2023 22:35:52 -0400 Subject: [PATCH 31/36] fix(cve): unpin yaml-language-server --- docker-bits/6_jupyterlab.Dockerfile | 5 +++-- output/jupyterlab-cpu/Dockerfile | 5 +++-- output/jupyterlab-pytorch/Dockerfile | 5 +++-- output/jupyterlab-tensorflow/Dockerfile | 5 +++-- output/sas/Dockerfile | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docker-bits/6_jupyterlab.Dockerfile b/docker-bits/6_jupyterlab.Dockerfile index 6f205ed3d..83446f301 100644 --- a/docker-bits/6_jupyterlab.Dockerfile +++ b/docker-bits/6_jupyterlab.Dockerfile @@ -94,8 +94,9 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ - 'yaml-language-server@0.18.0' && \ - clean-layer.sh && \ + 'yaml-language-server' && \ + npm audit fix --force && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 430516484..4cdcce621 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -245,8 +245,9 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ - 'yaml-language-server@0.18.0' && \ - clean-layer.sh && \ + 'yaml-language-server' && \ + npm audit fix --force && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index ffda8f01d..1622525ae 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -267,8 +267,9 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ - 'yaml-language-server@0.18.0' && \ - clean-layer.sh && \ + 'yaml-language-server' && \ + npm audit fix --force && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 037524e3a..38d3db8f8 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -374,8 +374,9 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ - 'yaml-language-server@0.18.0' && \ - clean-layer.sh && \ + 'yaml-language-server' && \ + npm audit fix --force && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 3cfab7788..086855181 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -247,8 +247,9 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ - 'yaml-language-server@0.18.0' && \ - clean-layer.sh && \ + 'yaml-language-server' && \ + npm audit fix --force && \ + clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER From e673763e600b164b0410a780f21d3fea6c84be5e Mon Sep 17 00:00:00 2001 From: souheil Date: Sun, 22 Oct 2023 23:02:25 -0400 Subject: [PATCH 32/36] fix(cve): remove audit fix --- .gitignore | 3 +++ docker-bits/6_jupyterlab.Dockerfile | 1 - output/jupyterlab-cpu/Dockerfile | 1 - output/jupyterlab-pytorch/Dockerfile | 1 - output/jupyterlab-tensorflow/Dockerfile | 1 - output/sas/Dockerfile | 1 - 6 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index b2b27547b..1696ccd57 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ __pycache__ .venv .ipynb_checkpoints/ .idea +node_modules +package-lock.json +package.json \ No newline at end of file diff --git a/docker-bits/6_jupyterlab.Dockerfile b/docker-bits/6_jupyterlab.Dockerfile index 83446f301..416bd4cea 100644 --- a/docker-bits/6_jupyterlab.Dockerfile +++ b/docker-bits/6_jupyterlab.Dockerfile @@ -95,7 +95,6 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ - npm audit fix --force && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 4cdcce621..85b127e41 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -246,7 +246,6 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ - npm audit fix --force && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index 1622525ae..70f2f3a49 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -268,7 +268,6 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ - npm audit fix --force && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 38d3db8f8..711a1a89c 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -375,7 +375,6 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ - npm audit fix --force && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 086855181..70bdc3751 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -248,7 +248,6 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ - npm audit fix --force && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER From d68774f3cebb7816f88965efc16b16f2a6c4371b Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 23 Oct 2023 00:12:14 -0400 Subject: [PATCH 33/36] feat(jupyterlab): add graphviz and try to patch babel --- docker-bits/3_Kubeflow.Dockerfile | 3 ++- docker-bits/6_jupyterlab.Dockerfile | 3 +++ output/jupyterlab-cpu/Dockerfile | 6 +++++- output/jupyterlab-pytorch/Dockerfile | 6 +++++- output/jupyterlab-tensorflow/Dockerfile | 6 +++++- output/remote-desktop/Dockerfile | 3 ++- output/rstudio/Dockerfile | 3 ++- output/sas/Dockerfile | 6 +++++- 8 files changed, 29 insertions(+), 7 deletions(-) diff --git a/docker-bits/3_Kubeflow.Dockerfile b/docker-bits/3_Kubeflow.Dockerfile index 08aa2c549..61fed3f5c 100644 --- a/docker-bits/3_Kubeflow.Dockerfile +++ b/docker-bits/3_Kubeflow.Dockerfile @@ -8,7 +8,8 @@ RUN mamba install --quiet \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ - 'fire==0.5.0' && \ + 'fire==0.5.0' \ + 'graphviz' && \ pip install 'kubeflow-training' && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ diff --git a/docker-bits/6_jupyterlab.Dockerfile b/docker-bits/6_jupyterlab.Dockerfile index 416bd4cea..d6c000923 100644 --- a/docker-bits/6_jupyterlab.Dockerfile +++ b/docker-bits/6_jupyterlab.Dockerfile @@ -89,12 +89,15 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'r-languageserver' \ 'python-lsp-server' \ && \ +# These should probably go in a package.json file +# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs npm i -g \ 'bash-language-server' \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ + npm uninstall -g @babel/traverse && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 85b127e41..6665bd532 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -42,7 +42,8 @@ RUN mamba install --quiet \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ - 'fire==0.5.0' && \ + 'fire==0.5.0' \ + 'graphviz' && \ pip install 'kubeflow-training' && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ @@ -240,12 +241,15 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'r-languageserver' \ 'python-lsp-server' \ && \ +# These should probably go in a package.json file +# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs npm i -g \ 'bash-language-server' \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ + npm uninstall -g @babel/traverse && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index 70f2f3a49..fabe5b51c 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -64,7 +64,8 @@ RUN mamba install --quiet \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ - 'fire==0.5.0' && \ + 'fire==0.5.0' \ + 'graphviz' && \ pip install 'kubeflow-training' && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ @@ -262,12 +263,15 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'r-languageserver' \ 'python-lsp-server' \ && \ +# These should probably go in a package.json file +# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs npm i -g \ 'bash-language-server' \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ + npm uninstall -g @babel/traverse && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index 711a1a89c..fe0073810 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -171,7 +171,8 @@ RUN mamba install --quiet \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ - 'fire==0.5.0' && \ + 'fire==0.5.0' \ + 'graphviz' && \ pip install 'kubeflow-training' && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ @@ -369,12 +370,15 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'r-languageserver' \ 'python-lsp-server' \ && \ +# These should probably go in a package.json file +# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs npm i -g \ 'bash-language-server' \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ + npm uninstall -g @babel/traverse && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/output/remote-desktop/Dockerfile b/output/remote-desktop/Dockerfile index d9f3ded7b..e7bd96ab7 100644 --- a/output/remote-desktop/Dockerfile +++ b/output/remote-desktop/Dockerfile @@ -118,7 +118,8 @@ RUN mamba install --quiet \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ - 'fire==0.5.0' && \ + 'fire==0.5.0' \ + 'graphviz' && \ pip install 'kubeflow-training' && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ diff --git a/output/rstudio/Dockerfile b/output/rstudio/Dockerfile index 833a8ea6f..92603ba54 100644 --- a/output/rstudio/Dockerfile +++ b/output/rstudio/Dockerfile @@ -42,7 +42,8 @@ RUN mamba install --quiet \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ - 'fire==0.5.0' && \ + 'fire==0.5.0' \ + 'graphviz' && \ pip install 'kubeflow-training' && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 70bdc3751..ca39990a7 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -44,7 +44,8 @@ RUN mamba install --quiet \ 'joblib==1.2.0' \ # s3 file system tool forked by Zach, ~4 years old, to be upgraded 's3fs' \ - 'fire==0.5.0' && \ + 'fire==0.5.0' \ + 'graphviz' && \ pip install 'kubeflow-training' && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ @@ -242,12 +243,15 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'r-languageserver' \ 'python-lsp-server' \ && \ +# These should probably go in a package.json file +# Copy the file over then use npm ci, much better flexibility for managing deps and CVEs npm i -g \ 'bash-language-server' \ 'dockerfile-language-server-nodejs' \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ + npm uninstall -g @babel/traverse && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER From 6bd7bbdfa8c514d215cc71a75eb971237ead856a Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 23 Oct 2023 09:57:06 -0400 Subject: [PATCH 34/36] fix(graphviz): add to test_packages exclusion --- tests/general/test_packages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/general/test_packages.py b/tests/general/test_packages.py index 79e9b5f89..589cf4612 100644 --- a/tests/general/test_packages.py +++ b/tests/general/test_packages.py @@ -95,6 +95,7 @@ "pillow", # import yaml, not import pyyaml "pyyaml", + "graphviz", ] From 59fc6601224c4a6745dd1251a6947286a10ab04e Mon Sep 17 00:00:00 2001 From: souheil Date: Mon, 23 Oct 2023 14:59:08 -0400 Subject: [PATCH 35/36] fix(cve): update openmpp UI --- docker-bits/6_jupyterlab.Dockerfile | 7 +++---- output/jupyterlab-cpu/Dockerfile | 7 +++---- output/jupyterlab-pytorch/Dockerfile | 7 +++---- output/jupyterlab-tensorflow/Dockerfile | 7 +++---- output/sas/Dockerfile | 7 +++---- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/docker-bits/6_jupyterlab.Dockerfile b/docker-bits/6_jupyterlab.Dockerfile index d6c000923..521993cd9 100644 --- a/docker-bits/6_jupyterlab.Dockerfile +++ b/docker-bits/6_jupyterlab.Dockerfile @@ -97,17 +97,16 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ - npm uninstall -g @babel/traverse && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER # OpenM install # Install OpenM++ MPI -ARG OMPP_VERSION="1.15.4" +ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! -ARG OMPP_PKG_DATE="20230803" -ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 +ARG OMPP_PKG_DATE="20231005" +ARG SHA256ompp=7b88490371875567bdc95aba3a3fc8c66a17f4c8c86e9a95b4f924d1c7292eb4 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 6665bd532..61c284dca 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -249,17 +249,16 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ - npm uninstall -g @babel/traverse && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER # OpenM install # Install OpenM++ MPI -ARG OMPP_VERSION="1.15.4" +ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! -ARG OMPP_PKG_DATE="20230803" -ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 +ARG OMPP_PKG_DATE="20231005" +ARG SHA256ompp=7b88490371875567bdc95aba3a3fc8c66a17f4c8c86e9a95b4f924d1c7292eb4 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index fabe5b51c..8df633a12 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -271,17 +271,16 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ - npm uninstall -g @babel/traverse && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER # OpenM install # Install OpenM++ MPI -ARG OMPP_VERSION="1.15.4" +ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! -ARG OMPP_PKG_DATE="20230803" -ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 +ARG OMPP_PKG_DATE="20231005" +ARG SHA256ompp=7b88490371875567bdc95aba3a3fc8c66a17f4c8c86e9a95b4f924d1c7292eb4 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index fe0073810..e627fab55 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -378,17 +378,16 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ - npm uninstall -g @babel/traverse && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER # OpenM install # Install OpenM++ MPI -ARG OMPP_VERSION="1.15.4" +ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! -ARG OMPP_PKG_DATE="20230803" -ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 +ARG OMPP_PKG_DATE="20231005" +ARG SHA256ompp=7b88490371875567bdc95aba3a3fc8c66a17f4c8c86e9a95b4f924d1c7292eb4 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index ca39990a7..0cc1d19f6 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -251,17 +251,16 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ 'javascript-typescript-langserver' \ 'unified-language-server' \ 'yaml-language-server' && \ - npm uninstall -g @babel/traverse && \ clean-layer.sh && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER # OpenM install # Install OpenM++ MPI -ARG OMPP_VERSION="1.15.4" +ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! -ARG OMPP_PKG_DATE="20230803" -ARG SHA256ompp=5da79984ef67ad16b3b7d429896b8a553930ca46a16079aaef24b3c9dc867956 +ARG OMPP_PKG_DATE="20231005" +ARG SHA256ompp=7b88490371875567bdc95aba3a3fc8c66a17f4c8c86e9a95b4f924d1c7292eb4 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} From 027317c6e4435064003b2afd52d66d9111b141bb Mon Sep 17 00:00:00 2001 From: souheil-yazji Date: Mon, 23 Oct 2023 19:16:24 +0000 Subject: [PATCH 36/36] fix(cve): update ompp checksum --- docker-bits/6_jupyterlab.Dockerfile | 2 +- output/jupyterlab-cpu/Dockerfile | 2 +- output/jupyterlab-pytorch/Dockerfile | 2 +- output/jupyterlab-tensorflow/Dockerfile | 2 +- output/sas/Dockerfile | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-bits/6_jupyterlab.Dockerfile b/docker-bits/6_jupyterlab.Dockerfile index 521993cd9..e5713b782 100644 --- a/docker-bits/6_jupyterlab.Dockerfile +++ b/docker-bits/6_jupyterlab.Dockerfile @@ -106,7 +106,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! ARG OMPP_PKG_DATE="20231005" -ARG SHA256ompp=7b88490371875567bdc95aba3a3fc8c66a17f4c8c86e9a95b4f924d1c7292eb4 +ARG SHA256ompp=6d44076e1890c2e2ffb431182b9565cb4715830a027b01aafb9531e274bb8e84 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} diff --git a/output/jupyterlab-cpu/Dockerfile b/output/jupyterlab-cpu/Dockerfile index 61c284dca..4288c5b58 100644 --- a/output/jupyterlab-cpu/Dockerfile +++ b/output/jupyterlab-cpu/Dockerfile @@ -258,7 +258,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! ARG OMPP_PKG_DATE="20231005" -ARG SHA256ompp=7b88490371875567bdc95aba3a3fc8c66a17f4c8c86e9a95b4f924d1c7292eb4 +ARG SHA256ompp=6d44076e1890c2e2ffb431182b9565cb4715830a027b01aafb9531e274bb8e84 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} diff --git a/output/jupyterlab-pytorch/Dockerfile b/output/jupyterlab-pytorch/Dockerfile index 8df633a12..efdcfc9c6 100644 --- a/output/jupyterlab-pytorch/Dockerfile +++ b/output/jupyterlab-pytorch/Dockerfile @@ -280,7 +280,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! ARG OMPP_PKG_DATE="20231005" -ARG SHA256ompp=7b88490371875567bdc95aba3a3fc8c66a17f4c8c86e9a95b4f924d1c7292eb4 +ARG SHA256ompp=6d44076e1890c2e2ffb431182b9565cb4715830a027b01aafb9531e274bb8e84 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} diff --git a/output/jupyterlab-tensorflow/Dockerfile b/output/jupyterlab-tensorflow/Dockerfile index e627fab55..b60c1dbf2 100644 --- a/output/jupyterlab-tensorflow/Dockerfile +++ b/output/jupyterlab-tensorflow/Dockerfile @@ -387,7 +387,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! ARG OMPP_PKG_DATE="20231005" -ARG SHA256ompp=7b88490371875567bdc95aba3a3fc8c66a17f4c8c86e9a95b4f924d1c7292eb4 +ARG SHA256ompp=6d44076e1890c2e2ffb431182b9565cb4715830a027b01aafb9531e274bb8e84 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION} diff --git a/output/sas/Dockerfile b/output/sas/Dockerfile index 0cc1d19f6..079639426 100644 --- a/output/sas/Dockerfile +++ b/output/sas/Dockerfile @@ -260,7 +260,7 @@ RUN julia -e 'using Pkg; Pkg.add("LanguageServer")' && \ ARG OMPP_VERSION="1.15.5" # IMPORTANT: Don't forget to update the version number in the openmpp.desktop file!! ARG OMPP_PKG_DATE="20231005" -ARG SHA256ompp=7b88490371875567bdc95aba3a3fc8c66a17f4c8c86e9a95b4f924d1c7292eb4 +ARG SHA256ompp=6d44076e1890c2e2ffb431182b9565cb4715830a027b01aafb9531e274bb8e84 # OpenM++ environment settings ENV OMPP_INSTALL_DIR=/opt/openmpp/${OMPP_VERSION}