ci_cd.yml

name: CI + CD

on:
  pull_request:
  push:
    branches:
      - main
  release:
    types:
      - published
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'release' && 'release' || github.ref }}
  # Don't continue building images for a PR if the PR is updated quickly
  # For other workflows, allow them to complete and just block on them. This
  # ensures deployments in particular to happen in series rather than parallel.
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  lint:
    name: Lint
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Cache pre-commit envs
        uses: actions/cache@v3
        with:
          path: ~/.cache/pre-commit
          key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}

      - name: Run pre-commit to lint files
        run: |
          pip install pre-commit
          pre-commit run --all-files

  get-image-tag:
    name: Get image tag
    runs-on: ubuntu-latest
    outputs:
      image-tag: ${{ steps.get-image-tag.outputs.image-tag }}
    steps:
      - name: Get image tag
        id: get-image-tag
        run: |
          if [[ "${{ github.event_name }}" == "release" ]]; then
              echo "image-tag=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
          else
              echo "image-tag=${{ github.sha }}" >> "$GITHUB_OUTPUT"
          fi

  build-images:
    name: Build images
    runs-on: ubuntu-latest
    strategy:
      matrix:
        image:
          - api
          - ingestion_server
    needs:
      - get-image-tag
      - lint
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v2
        with:
          install: true

      - name: Build image `${{ matrix.image }}`
        uses: docker/build-push-action@v4
        with:
          context: ${{ matrix.image }}
          push: false
          tags: openverse-${{ matrix.image }}
          cache-from: type=gha,scope=${{ matrix.image }}
          cache-to: type=gha,scope=${{ matrix.image }}
          outputs: type=docker,dest=/tmp/${{ matrix.image }}.tar
          build-args: |
            SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image-tag }}

      - name: Upload image `${{ matrix.image }}`
        uses: actions/upload-artifact@v3
        with:
          name: ${{ matrix.image }}
          path: /tmp/${{ matrix.image }}.tar

  build-nginx:
    # This requires a separate job due to the dependency on the other image builds
    name: Build `nginx` Dockerfile target
    runs-on: ubuntu-latest
    needs:
      - build-images
      - get-image-tag

    steps:
      - uses: actions/checkout@v3

      - name: Setup just
        uses: extractions/setup-just@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Download all images
        uses: actions/download-artifact@v3
        with:
          path: /tmp

      - name: Load API and ingestion server images
        run: |
          docker load --input /tmp/api/api.tar
          docker load --input /tmp/ingestion_server/ingestion_server.tar

      - name: collectstatic
        run: just collectstatic
        env:
          STATIC_ROOT: ./static
          DC_USER: root

      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v2
        with:
          install: true

      - name: Build image `nginx`
        uses: docker/build-push-action@v4
        with:
          context: api
          target: nginx
          push: false
          tags: openverse-api-nginx
          cache-from: type=gha,scope=nginx
          cache-to: type=gha,scope=nginx
          outputs: type=docker,dest=/tmp/api-nginx.tar
          build-args: |
            SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image-tag }}

      - name: Upload image `api-nginx`
        uses: actions/upload-artifact@v3
        with:
          name: api-nginx
          path: /tmp/api-nginx.tar

  test-ing:
    name: Run tests for ingestion-server
    runs-on: ubuntu-latest
    timeout-minutes: 15
    needs:
      - build-images
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Setup just
        uses: extractions/setup-just@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: Cache Python dependencies
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile.lock') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install Python deps for ingestion-server
        run: |
          pip install -U pip
          pip install pipenv
          just _ing-install

      - name: Download image `ingestion_server`
        uses: actions/download-artifact@v3
        with:
          name: ingestion_server
          path: /tmp
      - name: Load image `ingestion_server`
        run: docker load --input /tmp/ingestion_server.tar

      - name: Run ingestion-server tests
        run: just ing-testlocal

      - name: Upload ingestion test logs
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: ing_logs
          path: ingestion_server/test/ingestion_logs.txt

      - name: Print ingestion test logs
        if: always()
        run: cat ingestion_server/test/ingestion_logs.txt

  test-api:
    name: Run tests for the API
    runs-on: ubuntu-latest
    timeout-minutes: 15
    needs:
      - build-images
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Setup just
        uses: extractions/setup-just@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Download all images
        uses: actions/download-artifact@v3
        with:
          path: /tmp

      - name: Load all images
        run: |
          docker load --input /tmp/api/api.tar
          docker load --input /tmp/ingestion_server/ingestion_server.tar

      - name: Start API, ingest and index test data
        run: just init

      - name: Run API tests
        run: just api-test

      - name: Print API test logs
        if: always()
        run: |
          just logs > api_logs
          cat api_logs

      - name: Upload API test logs
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: api_logs
          path: api_logs

  test-redoc:
    name: Check for API consumer docs
    runs-on: ubuntu-latest
    needs:
      - build-images
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Setup just
        uses: extractions/setup-just@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Download all images
        uses: actions/download-artifact@v3
        with:
          path: /tmp

      - name: Load all images
        run: |
          docker load --input /tmp/api/api.tar
          docker load --input /tmp/ingestion_server/ingestion_server.tar

      - name: Test ReDoc site
        run: just api-doctest

  validate-openapi-spec:
    name: Validate Open API spec
    runs-on: ubuntu-latest
    needs:
      - build-images
    steps:
      - uses: actions/checkout@v3

      - name: Setup just
        uses: extractions/setup-just@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Download all images
        uses: actions/download-artifact@v3
        with:
          path: /tmp

      - name: Load all images
        run: |
          docker load --input /tmp/api/api.tar
          docker load --input /tmp/ingestion_server/ingestion_server.tar

      - name: Run check
        run: just dj validateopenapischema
        env:
          DC_USER: root

      - name: Upload schema on failure
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: openverse-api-openapi-schema.yaml
          path: ./api/openapi.yaml

  django-check:
    name: Run Django check
    runs-on: ubuntu-latest
    needs:
      - build-images
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Setup just
        uses: extractions/setup-just@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Download all images
        uses: actions/download-artifact@v3
        with:
          path: /tmp

      - name: Load all images
        run: |
          docker load --input /tmp/api/api.tar
          docker load --input /tmp/ingestion_server/ingestion_server.tar

      - name: Run check
        run: just dj check

  check-migrations:
    name: Check for uncommited Django migrations
    runs-on: ubuntu-latest
    needs:
      - build-images
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Setup just
        uses: extractions/setup-just@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Download all images
        uses: actions/download-artifact@v3
        with:
          path: /tmp

      - name: Load all images
        run: |
          docker load --input /tmp/api/api.tar
          docker load --input /tmp/ingestion_server/ingestion_server.tar

      - name: Run makemigrations
        run: just dj makemigrations --check --noinput --merge

  push:
    name: Publish Docker images
    runs-on: ubuntu-latest
    # prevent running on fork PRs
    if: (github.event_name == 'push' || github.event_name == 'release') && github.repository == 'WordPress/openverse-api'
    needs:
      - test-ing
      - test-api
      - get-image-tag
      - build-nginx
    permissions:
      packages: write
      contents: read
    strategy:
      matrix:
        image:
          - api
          - ingestion_server
          - api-nginx
    steps:
      - name: Log in to GitHub Docker Registry
        uses: docker/login-action@v2
        with:
          registry: https://ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Download image `${{ matrix.image }}`
        uses: actions/download-artifact@v3
        with:
          name: ${{ matrix.image }}
          path: /tmp

      - name: Load and tag image `${{ matrix.image }}` (latest & sha)
        run: |
          docker load --input /tmp/${{ matrix.image }}.tar
          docker tag openverse-${{ matrix.image }} \
            ghcr.io/wordpress/openverse-${{ matrix.image }}:latest
          docker tag openverse-${{ matrix.image }} \
            ghcr.io/wordpress/openverse-${{ matrix.image }}:${{ needs.get-image-tag.outputs.image-tag }}
          docker push --all-tags ghcr.io/wordpress/openverse-${{ matrix.image }}

  deploy-staging:
    name: Deploy staging API
    runs-on: ubuntu-latest
    # Deploy a new version when `main` is pushed to in this repository
    if: github.event_name == 'push' && github.repository == 'WordPress/openverse-api'
    needs:
      - push
      - get-image-tag
    steps:
      - uses: actions/checkout@v3

      - uses: ./.github/actions/staging-deploy
        with:
          tag: ${{ needs.get-image-tag.outputs.image-tag }}
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          slack-webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
          gh-slack-username-map: ${{ secrets.GH_SLACK_USERNAME_MAP }}

  publish-docs:
    name: Publish developer docs
    runs-on: ubuntu-latest
    # Push new docs when `main` is pushed to in this repository
    if: github.event_name == 'push' && github.repository == 'WordPress/openverse-api'
    needs:
      - build-images

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Setup just
        uses: extractions/setup-just@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Download all images
        uses: actions/download-artifact@v3
        with:
          path: /tmp

      - name: Load all images
        run: |
          docker load --input /tmp/api/api.tar
          docker load --input /tmp/ingestion_server/ingestion_server.tar

      - name: Make developer docs
        run: |
          just sphinx-make
          sudo chown "$USER:$USER" -R ./api
          mv ./api/build/html /tmp/gh-pages
        env:
          DC_USER: root

      - uses: actions/checkout@v3
        with:
          ref: gh-pages

      - name: Preserve previews
        run: mv _preview /tmp/gh-pages/_preview

      - name: Deploy
        uses: peaceiris/actions-gh-pages@bbdfb200618d235585ad98e965f4aafc39b4c501 # v3.7.3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: /tmp/gh-pages
          force_orphan: true

      - name: Checkout branch again to enable cleaning
        if: always()
        uses: actions/checkout@v3
        with:
          ref: main

  docs-preview:
    name: Publish preview of developer docs for the branch
    runs-on: ubuntu-latest
    # Check if the event is not triggered by a fork or dependabot
    if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.owner.login == 'WordPress' && github.actor != 'dependabot[bot]'
    needs:
      - build-images
    steps:
      - uses: peter-evans/find-comment@v2
        id: preview-comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          body-includes: API Developer Docs Preview

      - uses: peter-evans/create-or-update-comment@v2
        if: steps.preview-comment.outputs.comment-id != 0
        with:
          comment-id: ${{ steps.preview-comment.outputs.comment-id }}
          edit-mode: replace
          body: |
            **API Developer Docs Preview**: _Updating_
            (This comment will be automatically updated with the preview URL once it is ready)

      - uses: peter-evans/create-or-update-comment@v2
        if: steps.preview-comment.outputs.comment-id == 0
        with:
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            **API Developer Docs Preview**: _Updating_
            (This comment will be automatically updated with the preview URLs once it is ready)

      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Setup just
        uses: extractions/setup-just@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Download all images
        uses: actions/download-artifact@v3
        with:
          path: /tmp

      - name: Load all images
        run: |
          docker load --input /tmp/api/api.tar
          docker load --input /tmp/ingestion_server/ingestion_server.tar

      - name: Build docs
        run: |
          just sphinx-make
          sudo chown "$USER:$USER" -R ./api
          mv ./api/build/html /tmp/preview
        env:
          DC_USER: root

      # Otherwise we end up with any extra stuff not git-ignored in the gh-pages branch
      - name: Recreate working directory to avoid superfluous files
        run: |
          cd ..
          sudo rm -rf openverse-api
          mkdir openverse-api

      - uses: actions/checkout@v3
        with:
          ref: gh-pages

      - name: Merge preview with existing gh-pages
        run: |
          cd ..
          cp -r ./openverse-api /tmp/gh-pages
          # trash the existing preview folder and replace it with the newly generated one
          # if the preview hasn't been pushed yet this will still exit(0)
          sudo rm -rf /tmp/gh-pages/_preview/${{ github.event.pull_request.number }}
          mv /tmp/preview /tmp/gh-pages/_preview/${{ github.event.pull_request.number }}
          cd openverse-api

      - name: Deploy
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: /tmp/gh-pages
          force_orphan: true

      - uses: peter-evans/find-comment@v2
        id: final-preview-comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          body-includes: API Developer Docs Preview

      - uses: peter-evans/create-or-update-comment@v2
        with:
          comment-id: ${{ steps.final-preview-comment.outputs.comment-id }}
          edit-mode: replace
          body: |
            **API Developer Docs Preview**: _Ready_

            <https://wordpress.github.io/openverse-api/_preview/${{ github.event.pull_request.number }}>

            Please note that GitHub pages takes a little time to deploy newly pushed code, if the links above don't work or you see old versions, wait 5 minutes and try again.

            You can check [the GitHub pages deployment action list](https://github.com/WordPress/openverse-api/actions/workflows/pages/pages-build-deployment) to see the current status of the deployments.

      - name: Checkout branch again to enable cleaning
        uses: actions/checkout@v3
        if: always()