app-tests.yaml

name: App Tests

on:
  push:
    branches: [ main ]
    paths-ignore:
      - "**.md"
      - ".azdo/**"
      - ".devcontainer/**"
      - ".github/**"
  pull_request:
    branches: [ main ]
    paths-ignore:
      - "**.md"
      - ".azdo/**"
      - ".devcontainer/**"
      - ".github/**"
  workflow_dispatch:

permissions:
  contents: read

jobs:
  test-package:
    name: Test ${{ matrix.os }} Python ${{ matrix.python_version }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest", "macos-latest-xlarge", "macos-13", "windows-latest"]
        python_version: ["3.9", "3.10", "3.11", "3.12"]
        exclude:
          - os: macos-latest-xlarge
            python_version: "3.9"
          - os: macos-latest-xlarge
            python_version: "3.10"
    env:
      UV_SYSTEM_PYTHON: 1
      POSTGRES_HOST: localhost
      POSTGRES_USERNAME: postgres
      POSTGRES_PASSWORD: root
      POSTGRES_DATABASE: postgres
      POSTGRES_SSL: disable
    steps:
        - uses: actions/checkout@v4

        - name: (MacOS) Install postgreSQL and pgvector using brew
          if: matrix.os == 'macos-13' || matrix.os == 'macos-latest-xlarge'
          run: |
            brew install postgresql@14
            brew link --overwrite postgresql@14
            brew install pgvector
            brew services start postgresql@14 && sleep 1
            createuser -s ${{ env.POSTGRES_USERNAME }}
            psql -d postgres -c "ALTER USER ${{ env.POSTGRES_USERNAME }} WITH PASSWORD '${{ env.POSTGRES_PASSWORD }}'"
            psql -d postgres -c 'CREATE EXTENSION vector'

        - name: Install pgvector
          if: matrix.os == 'windows-latest'
          shell: cmd
          run: |
              call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
              cd %TEMP%
              git clone --branch v0.7.4 https://github.com/pgvector/pgvector.git
              cd pgvector
              nmake /NOLOGO /F Makefile.win
              nmake /NOLOGO /F Makefile.win install
              sc config postgresql-x64-14 start=auto
              net start postgresql-x64-14
              "%PGBIN%/psql" -d postgres -c "CREATE EXTENSION vector"

        - name: (Linux) Install pgvector and set password
          if: matrix.os == 'ubuntu-latest'
          run: |
            sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
            sudo apt-get install postgresql-14-pgvector
            sudo systemctl start postgresql
            sudo -u postgres psql -c "ALTER USER ${{ env.POSTGRES_USERNAME }} PASSWORD '${{ env.POSTGRES_PASSWORD }}'"
            sudo -u postgres psql -c 'CREATE EXTENSION vector'

        - name: Setup python
          uses: actions/setup-python@v5
          with:
            python-version: ${{ matrix.python_version }}
            architecture: x64

        - name: Install uv
          uses: astral-sh/setup-uv@v4
          with:
            enable-cache: true
            version: "0.4.20"
            cache-dependency-glob: "requirements**.txt"

        - name: Install dependencies
          run: |
            uv pip install -r requirements-dev.txt
            uv pip install -r evals/requirements.txt

        - name: Install app as editable app
          run: |
            uv pip install -e src/backend

        - name: Setup local database with seed data
          run: |
            python ./src/backend/fastapi_app/setup_postgres_database.py
            python ./src/backend/fastapi_app/setup_postgres_seeddata.py

        - name: Setup node
          uses: actions/setup-node@v4
          with:
            node-version: 18

        - name: Build frontend
          run: |
            cd ./src/frontend
            npm install
            npm run build

        - name: Setup mypy cache
          uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
          with:
            path: ./.mypy_cache
            key: mypy${{ matrix.os }}-${{ matrix.python_version }}-${{ hashFiles('requirements-dev.txt', 'src/backend/requirements.txt', 'src/backend/pyproject.toml') }}

        - name: Run MyPy
          run: python3 -m mypy .

        - name: Run Pytest
          run: python3 -m pytest -s -vv --cov --cov-fail-under=85

        - name: Run E2E tests with Playwright
          id: e2e
          run: |
            playwright install chromium --with-deps
            python3 -m pytest tests/e2e.py --tracing=retain-on-failure

        - name: Upload test artifacts
          if: ${{ failure() && steps.e2e.conclusion == 'failure' }}
          uses: actions/upload-artifact@v4
          with:
            name: playwright-traces${{ matrix.python_version }}
            path: test-results