build_and_push #3452
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow: | |
| # * Builds, tests, and scans all images | |
| # * (optionally) pushes the images to ACR | |
| # | |
| # This workflow triggers on: | |
| # * a push to master | |
| # * any create/synchronize to a PR (eg: any time you push an update to a PR). | |
| # | |
| # Image build/test/scan will run on any of the above events. | |
| # Image push will run only if: | |
| # * this is a push to master | |
| # * if the PR triggering this event has the label 'auto-deploy' | |
| # | |
| # To configure this workflow: | |
| # | |
| # 1. Set up the following secrets in your workspace: | |
| # a. REGISTRY_USERNAME with ACR username | |
| # b. REGISTRY_PASSWORD with ACR Password | |
| # c. AZURE_CREDENTIALS with the output of `az ad sp create-for-rbac --sdk-auth` | |
| # d. DEV_REGISTRY_USERNAME with the DEV ACR username | |
| # e. DEV_REGISTRY_PASSWORD with the DEV ACR Password | |
| # | |
| # 2. Change the values for the REGISTRY_NAME, CLUSTER_NAME, CLUSTER_RESOURCE_GROUP and NAMESPACE environment variables (below in build-push). | |
| name: build_and_push | |
| on: | |
| schedule: | |
| # Execute at 2am EST every day | |
| - cron: '0 21 * * *' | |
| push: | |
| branches: | |
| - 'master' | |
| pull_request: | |
| types: | |
| - 'opened' | |
| - 'synchronize' | |
| - 'reopened' | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| jobs: | |
| # Any checks that run pre-build | |
| pre-build-checks: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@master | |
| - name: Assert committed ./output folder matches `make generate-dockerfiles` output | |
| run: | | |
| sudo apt-get install --yes make | |
| make clean | |
| make generate-dockerfiles | |
| if ! git diff --quiet output/; then | |
| echo 'output folder and docker-bits/resources out of sync!' | |
| exit 1 | |
| fi | |
| build-push: | |
| env: | |
| REGISTRY_NAME: k8scc01covidacr | |
| DEV_REGISTRY_NAME: k8scc01covidacrdev | |
| CLUSTER_NAME: k8s-cancentral-01-covid-aks | |
| CLUSTER_RESOURCE_GROUP: k8s-cancentral-01-covid-aks | |
| LOCAL_REPO: localhost:5000 | |
| TRIVY_VERSION: "v0.57.0" | |
| TRIVY_DATABASES: '"ghcr.io/aquasecurity/trivy-db:2","public.ecr.aws/aquasecurity/trivy-db"' | |
| TRIVY_JAVA_DATABASES: '"ghcr.io/aquasecurity/trivy-java-db:1","public.ecr.aws/aquasecurity/trivy-java-db"' | |
| TRIVY_MAX_RETRIES: 5 | |
| TRIVY_RETRY_DELAY: 20 | |
| HADOLINT_VERSION: "2.12.0" | |
| ACTIONS_RUNNER_DEBUG: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| notebook: | |
| # TODO: Pull this from a settings file or Makefile, that way Make can have the same list | |
| # - docker-stacks-datascience-notebook # Debugging | |
| - rstudio | |
| - sas | |
| - jupyterlab-cpu | |
| - jupyterlab-pytorch | |
| # - jupyterlab-tensorflow removed from build. https://jirab.statcan.ca/browse/BTIS-421 | |
| - remote-desktop | |
| needs: pre-build-checks | |
| runs-on: ubuntu-latest | |
| services: | |
| registry: | |
| image: registry:2 | |
| ports: | |
| - 5000:5000 | |
| steps: | |
| - name: Set ENV variables for a PR containing the auto-deploy tag | |
| if: github.event_name == 'pull_request' && contains( github.event.pull_request.labels.*.name, 'auto-deploy') | |
| run: | | |
| echo "REGISTRY=k8scc01covidacrdev.azurecr.io" >> "$GITHUB_ENV" | |
| echo "IMAGE_VERSION=dev" >> "$GITHUB_ENV" | |
| - name: Set ENV variables for pushes to master | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/master' | |
| run: | | |
| echo "REGISTRY=k8scc01covidacr.azurecr.io" >> "$GITHUB_ENV" | |
| echo "IMAGE_VERSION=v1" >> "$GITHUB_ENV" | |
| echo "IS_LATEST=true" >> "$GITHUB_ENV" | |
| - uses: actions/checkout@master | |
| - name: Echo disk usage before clean up | |
| run: ./.github/scripts/echo_usage.sh | |
| - name: Free up all available disk space before building | |
| run: ./.github/scripts/cleanup_runner.sh | |
| - name: Echo disk usage before build start | |
| run: ./.github/scripts/echo_usage.sh | |
| - name: Get current notebook name | |
| id: notebook-name | |
| shell: bash | |
| run: | | |
| echo NOTEBOOK_NAME=${{ matrix.notebook }} >> $GITHUB_OUTPUT | |
| # Connect to Azure Container registry (ACR) | |
| - uses: azure/docker-login@v1 | |
| with: | |
| login-server: ${{ env.REGISTRY_NAME }}.azurecr.io | |
| username: ${{ secrets.REGISTRY_USERNAME }} | |
| password: ${{ secrets.REGISTRY_PASSWORD }} | |
| # Connect to Azure DEV Container registry (ACR) | |
| - uses: azure/docker-login@v1 | |
| with: | |
| login-server: ${{ env.DEV_REGISTRY_NAME }}.azurecr.io | |
| username: ${{ secrets.DEV_REGISTRY_USERNAME }} | |
| password: ${{ secrets.DEV_REGISTRY_PASSWORD }} | |
| # Image building/storing locally | |
| - name: Make Dockerfiles | |
| run: make generate-dockerfiles | |
| - name: Run Hadolint | |
| run: | | |
| sudo curl -L https://github.com/hadolint/hadolint/releases/download/v${{ env.HADOLINT_VERSION }}/hadolint-Linux-x86_64 --output hadolint | |
| sudo chmod +x hadolint | |
| ./hadolint output/${{ matrix.notebook }}/Dockerfile --no-fail | |
| # make build emits full_image_name, image_tag, and image_repo outputs | |
| - name: Build image | |
| id: build-image | |
| run: make build/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }} | |
| - name: Echo disk usage after build completion | |
| run: ./.github/scripts/echo_usage.sh | |
| - name: Add standard tag names (short sha, sha, and branch) and any other post-build activity | |
| run: make post-build/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }} | |
| - name: Push image to local registry (default pushes all tags) | |
| run: make push/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }} | |
| # Image testing | |
| - name: Set Up Python for Test Suite | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Set up venv for Test Suite | |
| run: | | |
| python -m pip install --upgrade pip | |
| make install-python-dev-venv | |
| - name: Test image | |
| run: make test/${{ matrix.notebook }} REPO=${{ env.LOCAL_REPO }} | |
| # Free up space from build process (containerscan action will run out of space if we don't) | |
| - run: ./.github/scripts/cleanup_runner.sh | |
| # Scan image for vulnerabilities | |
| - name: Aqua Security Trivy image scan | |
| # see https://github.com/StatCan/aaw-private/issues/11 -- should be re-enabled | |
| if: steps.notebook-name.outputs.NOTEBOOK_NAME != 'sas' | |
| run: | | |
| printf ${{ secrets.CVE_ALLOWLIST }} > .trivyignore | |
| curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin ${{ env.TRIVY_VERSION }} | |
| set +e | |
| for ((i=0; i<${{ env.TRIVY_MAX_RETRIES }}; i++)); do | |
| echo "Attempt $((i + 1)) of ${{ env.TRIVY_MAX_RETRIES }}..." | |
| trivy image \ | |
| --db-repository ${{ env.TRIVY_DATABASES }} \ | |
| --java-db-repository ${{ env.TRIVY_JAVA_DATABASES }} \ | |
| ${{ steps.build-image.outputs.full_image_name }} \ | |
| --exit-code 10 --timeout=20m --scanners vuln --severity CRITICAL | |
| EXIT_CODE=$? | |
| if [[ $EXIT_CODE -eq 0 ]]; then | |
| echo "Trivy scan completed successfully." | |
| exit 0 | |
| elif [[ $EXIT_CODE -eq 10 ]]; then | |
| echo "Trivy scan completed successfully. Some vulnerabilities were found." | |
| exit 10 | |
| elif [[ $i -lt $(( ${{ env.TRIVY_MAX_RETRIES }} - 1)) ]]; then | |
| echo "Encountered unexpected error. Retrying in ${{ env.TRIVY_RETRY_DELAY }} seconds..." | |
| sleep ${{ env.TRIVY_RETRY_DELAY }} | |
| else | |
| echo "Unexpected error persists after ${{ env.TRIVY_MAX_RETRIES }} attempts. Exiting." | |
| exit 1 | |
| fi | |
| done | |
| # Push image to ACR | |
| # Pushes if this is a push to master or an update to a PR that has auto-deploy label | |
| - name: Test if we should push to ACR | |
| id: should-i-push | |
| if: | | |
| github.event_name == 'push' || | |
| ( | |
| github.event_name == 'pull_request' && | |
| contains( github.event.pull_request.labels.*.name, 'auto-deploy') | |
| ) | |
| run: echo 'boolean=true' >> $GITHUB_OUTPUT | |
| # Pull the local image back, then "build" it (will just tag the pulled image) | |
| - name: Pull image back from local repo | |
| if: steps.should-i-push.outputs.boolean == 'true' | |
| run: docker pull ${{ steps.build-image.outputs.full_image_name }} | |
| # Rename the localhost:5000/imagename:tag built above to use the real repo | |
| # (get above's name from build-image's output) | |
| - name: Tag images with real repository | |
| if: steps.should-i-push.outputs.boolean == 'true' | |
| run: > | |
| make post-build/${{ matrix.notebook }} DEFAULT_REPO=$REGISTRY IS_LATEST=$IS_LATEST | |
| IMAGE_VERSION=$IMAGE_VERSION SOURCE_FULL_IMAGE_NAME=${{ steps.build-image.outputs.full_image_name }} | |
| - name: Push image to registry | |
| if: steps.should-i-push.outputs.boolean == 'true' | |
| run: | | |
| make push/${{ matrix.notebook }} DEFAULT_REPO=$REGISTRY | |
| - name: Slack Notification | |
| if: failure() && github.event_name=='schedule' | |
| uses: act10ns/slack@v1 | |
| with: | |
| status: failure | |
| message: Build failed. https://github.com/StatCan/aaw-kubeflow-containers/actions/runs/${{github.run_id}} |