system-tests-and-deploy.yml

name: System tests and deploy

on:
  push:
    branches:
      - master
      - canary
  pull_request:
    branches:
      - master

concurrency:
  # New commits cancel previous builds only on pull requests. This is because `github.head_ref`` is only set on pull requests and `github.sha`` is unique for each commit.
  group: system-tests-${{ github.head_ref || github.sha }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@v2

      # This would cause all other tests to be skipped
      - name: Make sure we have no focused system-tests
        run: |
          test "$(grep -r 'test.only' ./system-tests/src/ | wc -l)" -eq 0


      - name: Free up disk space
        run: |
          echo "Space before:"
          df -h
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          echo "Space after:"
          df -h

      # If glcoud cli is installed skaffold spams the output because we're not logged in and in some cases might even fail the build.
      - name: Hide gcloud cli from skaffold
        run: sudo mv /usr/bin/gcloud /tmp/gcloud

      - name: Install skaffold
        uses: yokawasa/action-setup-kube-tools@v0.9.3
        with:
          skaffold: "2.8.0"

      - name: Run npm ci so that shared module gets copied
        run: npm ci

      - name: Download tmc-langs
        run: bin/download-tmc-langs

      - name: Build
        run: bin/skaffold-build-test-env-and-serialize-output

      - name: Store skaffold output json
        uses: actions/upload-artifact@v2
        with:
          name: skaffold-build-output
          path: ./skaffold-build-output.json

      - name: Export production images
        run: bin/skaffold-export-production-images

      - name: Store built images
        uses: actions/upload-artifact@v2
        with:
          name: skaffold-images
          path: ./skaffold-images
          retention-days: 5

      # If glcoud cli is installed skaffold spams the output because we're not logged in and in some cases might even fail the build.
      - name: Unhide gcloud cli if master
        run: sudo mv /tmp/gcloud /usr/bin/gcloud
        if: ${{ github.ref == 'refs/heads/master' }}

      - name: Authenticate to Google Cloud if master
        env:
          GCLOUD_SERVICE_KEY: ${{ secrets.GCLOUD_SERVICE_KEY }}
        run: |
          echo Authenticating to Google Cloud
          echo "$GCLOUD_SERVICE_KEY" | python -m base64 -d > /tmp/key.json
          gcloud auth activate-service-account --key-file=/tmp/key.json
          if gcloud auth configure-docker -q; then
              echo "Authenticated to Google Cloud..."
          else
              echo "Authentication to Google Cloud failed. Exiting..."
              exit 1
          fi
        if: ${{ github.ref == 'refs/heads/master' }}

      # These images are used for deployments and caching
      - name: Push images if master
        run: skaffold build -p push-images --filename ./skaffold.production.yaml
        if: ${{ github.ref == 'refs/heads/master' }}
      - name: Push latest images if master
        run: skaffold build -p push-images,latest-tag --filename ./skaffold.production.yaml
        if: ${{ github.ref == 'refs/heads/master' }}
      - name: Push images tagged with sha if master
        run: |
          # shellcheck disable=SC2016
          jq --raw-output '.builds[].tag' < skaffold-build-output.json | xargs -d $'\n' sh -c 'for arg do echo "> docker push $arg"; docker push "$arg"; done'
        if: ${{ github.ref == 'refs/heads/master' }}

  system-tests:
    needs: build
    runs-on: ubuntu-latest
    strategy:
      matrix:
        shard: [1, 2]
    steps:
      - name: Free up disk space
        run: |
          echo "Space before:"
          df -h
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          echo "Space after:"
          df -h
      - name: Check out repository code
        uses: actions/checkout@v2

        # This would cause all other tests to be skipped
      - name: Make sure we have no focused system-tests
        run: |
          test "$(grep -r 'test.only' ./system-tests/src/ | wc -l)" -eq 0

      - name: Install kustomize, kubectl & skaffold
        uses: yokawasa/action-setup-kube-tools@v0.9.3
        with:
          kubectl: "1.28.2"
          kustomize: "5.1.1"
          skaffold: "2.8.0"
      - name: Workaround for kube-proxy set nf_conntrack_max permission denied
        run: |
          # Workarounds this issue when starting up kube-system/kube-proxy:
          # I0719 17:18:25.711233       1 conntrack.go:100] Set sysctl 'net/netfilter/nf_conntrack_max' to 131072
          # F0719 17:18:25.711251       1 server.go:489] open /proc/sys/net/netfilter/nf_conntrack_max: permission denied
          sudo sysctl net/netfilter/nf_conntrack_max=131072

      - name: Install, start & configure minikube
        uses: manusa/actions-setup-minikube@v2.7.1
        with:
          minikube version: "v1.28.0"
          kubernetes version: "v1.25.3"
          start args: "--addons ingress --memory max"
          driver: "docker"
          github token: ${{ secrets.GITHUB_TOKEN }}

      - name: Debug kube-system
        run: |
          kubectl get pods --namespace kube-system
          PODS=$(kubectl get pods --namespace kube-system -o name | cut -d '/' -f 2)
          for pod in $PODS
          do
            echo "------------------"
            kubectl describe pod "$pod" --namespace kube-system
          done

      - name: Debug ingress
        run: |
          kubectl get pods --namespace ingress-nginx
          PODS=$(kubectl get pods --namespace ingress-nginx -o name | cut -d '/' -f 2)
          for pod in $PODS
          do
            echo "------------------"
            kubectl describe pod "$pod" --namespace ingress-nginx
          done

      - name: Run npm ci for shared-modules
        run: npm ci

      - name: Download skaffold build output artifact
        uses: actions/download-artifact@v2
        with:
          name: skaffold-build-output
          path: skaffold-build-output

      - name: Download built images
        uses: actions/download-artifact@v2
        with:
          name: skaffold-images
          path: skaffold-images

      - name: Import downloaded images to minikube
        run: bin/skaffold-import-images-to-minikube

      - name: Deploy to local minikube
        run: skaffold deploy --force --filename ./skaffold.production.yaml --build-artifacts=./skaffold-build-output/skaffold-build-output.json

      - name: Configure ingress
        run: echo "$(minikube ip) project-331.local" | sudo tee --append /etc/hosts

      - name: Setup system test environment
        run: npm ci
        working-directory: ./system-tests/

      - name: Run system tests
        run: npm run test -- -- --shard="${{ matrix.shard }}/2"
        working-directory: ./system-tests/

      # To prevent accidentally introducing flaky tests in a pr. Skipped on master in order to not slow down deploys.
      - name: Run system tests again to make sure they're stable (if not master)
        run: npm run test -- -- --shard="${{ matrix.shard }}/2"
        working-directory: ./system-tests/
        if: ${{ github.ref != 'refs/heads/master' }}

      - name: Headless lms logs
        run: kubectl logs 'deployment/headless-lms'
        if: failure()

      - uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: test-results
          path: ./system-tests/test-results/
          if-no-files-found: ignore

      - uses: actions/upload-artifact@v2
        if: failure()
        with:
          name: playwright-html-report
          path: ./system-tests/playwright-report/index.html
          if-no-files-found: ignore

  deploy:
    needs: system-tests
    if: ${{ github.ref == 'refs/heads/master' }}
    runs-on: ubuntu-latest
    environment:
      name: production
      url: https://courses.mooc.fi
    steps:
      - name: Check out repository code
        uses: actions/checkout@v2
      - name: Download skaffold build output artifact
        uses: actions/download-artifact@v2
        with:
          name: skaffold-build-output
          path: skaffold-build-output

      - name: Install kustomize, kubectl & skaffold
        uses: yokawasa/action-setup-kube-tools@v0.7.1
        with:
          kubectl: "1.25.4"
          kustomize: "4.5.7"
          skaffold: "2.0.1"
      - uses: google-github-actions/setup-gcloud@94337306dda8180d967a56932ceb4ddcf01edae7
        with:
          service_account_key: ${{ secrets.GCLOUD_SERVICE_KEY_DEPLOY_TO_PRODUCTION }}
          project_id: ${{ secrets.GKE_PROJECT }}
      - run: |-
          gcloud --quiet auth configure-docker

      # Get the GKE credentials so we can deploy to the cluster
      - uses: google-github-actions/get-gke-credentials@fb08709ba27618c31c09e014e1d8364b02e5042e
        with:
          cluster_name: ${{ secrets.GKE_CLUSTER }}
          location: ${{ secrets.GKE_ZONE }}
          credentials: ${{ secrets.GCLOUD_SERVICE_KEY_DEPLOY_TO_PRODUCTION }}

      - name: Masking
        run: |
          IPS=$(kubectl config view | grep server | tr -s ' ' | cut -d ' ' -f 3 | xargs echo)
          echo "::add-mask::$IPS"

      - name: Deploy with Skaffold
        run: skaffold deploy --force --filename ./skaffold.production.yaml -p production --build-artifacts=./skaffold-build-output/skaffold-build-output.json --namespace courses-moocfi

  typecheck:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@v2

      - name: Run npm ci in repo root
        run: npm ci

      - name: Run npm ci system-tests
        run: npm ci
        working-directory: ./system-tests/

      - name: Run tsc
        run: npx tsc --noEmit
        working-directory: ./system-tests/