tests-ha.yml

name: High Availability Tests

on:
  workflow_dispatch:
    inputs: {}
  workflow_run:
    workflows: ["Tests"]
    types:
      - completed

jobs:
  build:
    runs-on: ubuntu-latest
    if: github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch'

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        submodules: false

    - uses: actions/checkout@v4
      with:
        fetch-depth: 50
        submodules: true

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.12.2'

    # Build virtualenv

    - name: Handle virtualenv
      uses: syphar/restore-virtualenv@v1
      id: venv-cache
      with:
        requirement_files: |
          setup.py
          pyproject.toml
        custom_cache_key_element: v2

    - name: Install Python dependencies
      if: steps.venv-cache.outputs.cache-hit != 'true'
      shell: bash
      run: |
        set -e
        python -m pip install -U pip setuptools wheel build
        bdeps_script="import build; print('\n'.join(build.ProjectBuilder('.').build_system_requires))"
        readarray -t build_deps < <(python -c "${bdeps_script}")
        python -m pip download --dest=$VIRTUAL_ENV/deps "${build_deps[@]}" .[test]
        python -m pip install -U --no-index --find-links=$VIRTUAL_ENV/deps $VIRTUAL_ENV/deps/*

    # Prepare environment variables and shared artifacts

    # Our HA tests currently only work on Postgres 14 (see #6332),
    # so check it out before we compute our build cache keys.
    - name: Switch back to Postgres 14
      shell: bash
      run: |
        set -e
        cd postgres
        # Fetch postgres 14, since the clone was shallow
        git fetch origin REL_14_8 --depth=1
        # For whatever reason the tag doesn't get fetched, so find it
        # at FETCH_HEAD
        git checkout FETCH_HEAD

    - name: Compute cache keys
      env:
        GIST_TOKEN: ${{ secrets.CI_BOT_GIST_TOKEN }}
      run: |
        mkdir -p .tmp
        python setup.py -q ci_helper --type cli > .tmp/edgedbcli_git_rev.txt
        python setup.py -q ci_helper --type rust >.tmp/rust_cache_key.txt
        python setup.py -q ci_helper --type ext >.tmp/ext_cache_key.txt
        python setup.py -q ci_helper --type parsers >.tmp/parsers_cache_key.txt
        python setup.py -q ci_helper --type postgres >.tmp/postgres_git_rev.txt
        echo 'v0.17.0' >.tmp/stolon_git_rev.txt
        python setup.py -q ci_helper --type bootstrap >.tmp/bootstrap_cache_key.txt
        echo EDGEDBCLI_GIT_REV=$(cat .tmp/edgedbcli_git_rev.txt) >> $GITHUB_ENV
        echo POSTGRES_GIT_REV=$(cat .tmp/postgres_git_rev.txt) >> $GITHUB_ENV
        echo STOLON_GIT_REV=$(cat .tmp/stolon_git_rev.txt) >> $GITHUB_ENV
        echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV
        echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV

    - name: Upload shared artifacts
      uses: actions/upload-artifact@v4
      with:
        name: shared-artifacts
        path: .tmp
        retention-days: 1

    # Restore binary cache

    - name: Handle cached EdgeDB CLI binaries
      uses: actions/cache@v4
      id: cli-cache
      with:
        path: build/cli
        key: edb-cli-v3-${{ env.EDGEDBCLI_GIT_REV }}

    - name: Handle cached Rust extensions
      uses: actions/cache@v4
      id: rust-cache
      with:
        path: build/rust_extensions
        key: edb-rust-v3-${{ hashFiles('.tmp/rust_cache_key.txt') }}
        restore-keys: |
          edb-rust-v3-

    - name: Handle cached Cython extensions
      uses: actions/cache@v4
      id: ext-cache
      with:
        path: build/extensions
        key: edb-ext-v4-${{ hashFiles('.tmp/ext_cache_key.txt') }}

    - name: Handle cached PostgreSQL build
      uses: actions/cache@v4
      id: postgres-cache
      with:
        path: build/postgres/install
        key: edb-postgres-v2-${{ env.POSTGRES_GIT_REV }}

    - name: Handle cached Stolon build
      uses: actions/cache@v4
      id: stolon-cache
      with:
        path: build/stolon/bin
        key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}

    # Install system dependencies for building

    - name: Install system deps
      if: |
        steps.cli-cache.outputs.cache-hit != 'true' ||
        steps.rust-cache.outputs.cache-hit != 'true' ||
        steps.ext-cache.outputs.cache-hit != 'true' ||
        steps.stolon-cache.outputs.cache-hit != 'true' ||
        steps.postgres-cache.outputs.cache-hit != 'true'
      run: |
        sudo apt-get update
        sudo apt-get install -y uuid-dev libreadline-dev bison flex

    - name: Install rust toolchain
      if: |
        steps.cli-cache.outputs.cache-hit != 'true' ||
        steps.rust-cache.outputs.cache-hit != 'true'
      uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248
      with:
        components: "cargo,rustc,rust-std"
        toolchain: "stable"

    # Build EdgeDB CLI

    - name: Handle EdgeDB CLI build cache
      uses: actions/cache@v4
      if: steps.cli-cache.outputs.cache-hit != 'true'
      with:
        path: ${{ env.BUILD_TEMP }}/rust/cli
        key: edb-cli-build-v7-${{ env.EDGEDBCLI_GIT_REV }}
        restore-keys: |
          edb-cli-build-v7-

    - name: Build EdgeDB CLI
      env:
        CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/cli/cargo_home
        CACHE_HIT: ${{ steps.cli-cache.outputs.cache-hit }}
      run: |
        if [[ "$CACHE_HIT" == "true" ]]; then
          cp -v build/cli/bin/edgedb edb/cli/edgedb
        else
          python setup.py -v build_cli
        fi

    # Build Rust extensions

    - name: Handle Rust extensions build cache
      uses: actions/cache@v4
      if: steps.rust-cache.outputs.cache-hit != 'true'
      with:
        path: ${{ env.BUILD_TEMP }}/rust/extensions
        key: edb-rust-build-v1-${{ hashFiles('.tmp/rust_cache_key.txt') }}
        restore-keys: |
          edb-rust-build-v1-

    - name: Build Rust extensions
      env:
        CARGO_HOME: ${{ env.BUILD_TEMP }}/rust/extensions/cargo_home
        CACHE_HIT: ${{ steps.rust-cache.outputs.cache-hit }}
      run: |
        if [[ "$CACHE_HIT" != "true" ]]; then
          rm -rf ${BUILD_LIB}
          mkdir -p build/rust_extensions
          rsync -av ./build/rust_extensions/ ${BUILD_LIB}/
          python setup.py -v build_rust
          rsync -av ${BUILD_LIB}/ build/rust_extensions/
          rm -rf ${BUILD_LIB}
        fi
        rsync -av ./build/rust_extensions/edb/ ./edb/

    # Build libpg_query

    - name: Build libpg_query
      run: |
        python setup.py build_libpg_query

    # Build extensions

    - name: Handle Cython extensions build cache
      uses: actions/cache@v4
      if: steps.ext-cache.outputs.cache-hit != 'true'
      with:
        path: ${{ env.BUILD_TEMP }}/edb
        key: edb-ext-build-v3-${{ hashFiles('.tmp/ext_cache_key.txt') }}

    - name: Build Cython extensions
      env:
        CACHE_HIT: ${{ steps.ext-cache.outputs.cache-hit }}
        BUILD_EXT_MODE: py-only
      run: |
        if [[ "$CACHE_HIT" != "true" ]]; then
          rm -rf ${BUILD_LIB}
          mkdir -p ./build/extensions
          rsync -av ./build/extensions/ ${BUILD_LIB}/
          BUILD_EXT_MODE=py-only python setup.py -v build_ext
          rsync -av ${BUILD_LIB}/ ./build/extensions/
          rm -rf ${BUILD_LIB}
        fi
        rsync -av ./build/extensions/edb/ ./edb/

    # Build parsers

    - name: Handle compiled parsers cache
      uses: actions/cache@v4
      id: parsers-cache
      with:
        path: build/lib
        key: edb-parsers-v3-${{ hashFiles('.tmp/parsers_cache_key.txt') }}
        restore-keys: |
          edb-parsers-v3-

    - name: Build parsers
      env:
        CACHE_HIT: ${{ steps.parsers-cache.outputs.cache-hit }}
      run: |
        if [[ "$CACHE_HIT" != "true" ]]; then
          rm -rf ${BUILD_LIB}
          mkdir -p ./build/lib
          rsync -av ./build/lib/ ${BUILD_LIB}/
          python setup.py -v build_parsers
          rsync -av ${BUILD_LIB}/ ./build/lib/
          rm -rf ${BUILD_LIB}
        fi
        rsync -av ./build/lib/edb/ ./edb/

    # Build PostgreSQL

    - name: Build PostgreSQL
      env:
        CACHE_HIT: ${{ steps.postgres-cache.outputs.cache-hit }}
      run: |
        if [[ "$CACHE_HIT" == "true" ]]; then
          cp build/postgres/install/stamp build/postgres/
        else
          python setup.py build_postgres
          cp build/postgres/stamp build/postgres/install/
        fi

    # Build Stolon

    - name: Set up Go
      if: steps.stolon-cache.outputs.cache-hit != 'true'
      uses: actions/setup-go@v2
      with:
        go-version: 1.16

    - uses: actions/checkout@v4
      if: steps.stolon-cache.outputs.cache-hit != 'true'
      with:
        repository: sorintlab/stolon
        path: build/stolon
        ref: ${{ env.STOLON_GIT_REV }}
        fetch-depth: 0
        submodules: false

    - name: Build Stolon
      if: steps.stolon-cache.outputs.cache-hit != 'true'
      run: |
        mkdir -p build/stolon/bin/
        curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul
        chmod +x build/stolon/bin/consul
        cd build/stolon && make

    # Install edgedb-server and populate egg-info

    - name: Install edgedb-server and populate egg-info
      env:
        CACHE_HIT: ${{ steps.venv-cache.outputs.cache-hit }}
        BUILD_EXT_MODE: skip
      run: |
        if [[ "$CACHE_HIT" == "true" ]]; then
          rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/
        else
          # --no-build-isolation because we have explicitly installed all deps
          # and don't want them to be reinstalled in an "isolated env".
          pip install --no-build-isolation --no-deps -e .[test,docs]
          rsync -av ./edgedb_server.egg-info/ $VIRTUAL_ENV/edgedb_server.egg-info/
        fi

    # Refresh the bootstrap cache

    - name: Handle bootstrap cache
      uses: actions/cache@v4
      id: bootstrap-cache
      with:
        path: build/cache
        key: edb-bootstrap-v2-${{ hashFiles('.tmp/bootstrap_cache_key.txt') }}
        restore-keys: |
          edb-bootstrap-v2-

    - name: Bootstrap EdgeDB Server
      if: steps.bootstrap-cache.outputs.cache-hit != 'true'
      run: |
        edb server --bootstrap-only

  ha-test:
    needs: build
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        submodules: false

    - uses: actions/checkout@v4
      with:
        fetch-depth: 50
        submodules: true

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.12.2'

    - name: Handle virtualenv
      uses: syphar/restore-virtualenv@v1
      id: venv-cache
      with:
        requirement_files: |
          setup.py
          pyproject.toml
        custom_cache_key_element: v2

    # Restore the artifacts and environment variables

    - name: Download shared artifacts
      uses: Wandalen/wretry.action@a163f62ae554a8f3cbe27b23db15b60c0ae2e93c  # v1.3.0
      with:
        action: actions/download-artifact@v4
        with: |
          name: shared-artifacts
          path: .tmp

    - name: Set environment variables
      run: |
        echo EDGEDBCLI_GIT_REV=$(cat .tmp/edgedbcli_git_rev.txt) >> $GITHUB_ENV
        echo POSTGRES_GIT_REV=$(cat .tmp/postgres_git_rev.txt) >> $GITHUB_ENV
        echo STOLON_GIT_REV=$(cat .tmp/stolon_git_rev.txt) >> $GITHUB_ENV
        echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV
        echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV

    # Restore build cache

    - name: Restore cached EdgeDB CLI binaries
      uses: actions/cache@v4
      id: cli-cache
      with:
        path: build/cli
        key: edb-cli-v3-${{ env.EDGEDBCLI_GIT_REV }}

    - name: Restore cached Rust extensions
      uses: actions/cache@v4
      id: rust-cache
      with:
        path: build/rust_extensions
        key: edb-rust-v3-${{ hashFiles('.tmp/rust_cache_key.txt') }}

    - name: Restore cached Cython extensions
      uses: actions/cache@v4
      id: ext-cache
      with:
        path: build/extensions
        key: edb-ext-v4-${{ hashFiles('.tmp/ext_cache_key.txt') }}

    - name: Restore compiled parsers cache
      uses: actions/cache@v4
      id: parsers-cache
      with:
        path: build/lib
        key: edb-parsers-v3-${{ hashFiles('.tmp/parsers_cache_key.txt') }}

    - name: Restore cached PostgreSQL build
      uses: actions/cache@v4
      id: postgres-cache
      with:
        path: build/postgres/install
        key: edb-postgres-v2-${{ env.POSTGRES_GIT_REV }}

    - name: Restore cached Stolon build
      uses: actions/cache@v4
      id: stolon-cache
      with:
        path: build/stolon/bin
        key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}

    - name: Restore bootstrap cache
      uses: actions/cache@v4
      id: bootstrap-cache
      with:
        path: build/cache
        key: edb-bootstrap-v2-${{ hashFiles('.tmp/bootstrap_cache_key.txt') }}

    - name: Stop if we cannot retrieve the cache
      if: |
        steps.venv-cache.outputs.cache-hit != 'true' ||
        steps.cli-cache.outputs.cache-hit != 'true' ||
        steps.rust-cache.outputs.cache-hit != 'true' ||
        steps.ext-cache.outputs.cache-hit != 'true' ||
        steps.parsers-cache.outputs.cache-hit != 'true' ||
        steps.postgres-cache.outputs.cache-hit != 'true' ||
        steps.stolon-cache.outputs.cache-hit != 'true' ||
        steps.bootstrap-cache.outputs.cache-hit != 'true'
      run: |
        echo ::error::Cannot retrieve build cache.
        exit 1

    - name: Restore cache into the source tree
      run: |
        cp -v build/cli/bin/edgedb edb/cli/edgedb
        rsync -av ./build/rust_extensions/edb/ ./edb/
        rsync -av ./build/extensions/edb/ ./edb/
        rsync -av ./build/lib/edb/ ./edb/
        cp build/postgres/install/stamp build/postgres/
        rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/

    # Run the test

    - name: Test
      env:
        SHARD: ${{ matrix.shard }}
        EDGEDB_TEST_HA: 1
        EDGEDB_TEST_CONSUL_PATH: build/stolon/bin/consul
        EDGEDB_TEST_STOLON_CTL: build/stolon/bin/stolonctl
        EDGEDB_TEST_STOLON_SENTINEL: build/stolon/bin/stolon-sentinel
        EDGEDB_TEST_STOLON_KEEPER: build/stolon/bin/stolon-keeper
      run: |
        edb test -j1 -v -k test_ha_


  workflow-notifications:
    if: failure() && github.event_name != 'pull_request'
    name: Notify in Slack on failures
    needs:
      - build
      - ha-test
    runs-on: ubuntu-latest
    permissions:
      actions: 'read'
    steps:
      - name: Slack Workflow Notification
        uses: Gamesight/slack-workflow-status@26a36836c887f260477432e4314ec3490a84f309
        with:
          repo_token: ${{secrets.GITHUB_TOKEN}}
          slack_webhook_url: ${{secrets.ACTIONS_SLACK_WEBHOOK_URL}}
          name: 'Workflow notifications'
          icon_emoji: ':hammer:'
          include_jobs: 'on-failure'