ci.yml

name: ci

on:
  push:
    # A branch github-ci-updates can be created and used for ci
    # experiments and tweaks.
    branches: [ "develop", "master", "github-ci", "windows", "pr_413_continued_plugins" ]
  pull_request:
    branches: [ "develop", "master" ]

permissions:
  contents: read

jobs:

  # Run all tests.
  base-ci:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    strategy:
      matrix:
        python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]

    steps:

    - name: Install Dependencies
      run: |
        sudo apt-get update -y
        sudo apt-get install -y mecab mecab-ipadic-utf8
        # Helper checks to find the mecab library path,
        # so it can be exported before running tests.
        # Without the export, natto-py fails on github.
        # echo FIND THE LIB:
        # which mecab
        # actual=`readlink -f /usr/bin/mecab`
        # ldd $actual

    - uses: actions/checkout@v4
      with:
        submodules: true

    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python_version }}
        cache: 'pip' # caching pip dependencies

    - run: pip install -r requirements-dev.txt

    - name: Setup config
      run: |
        mkdir ${{ github.workspace }}/data
        echo "ENV: dev" > ${{ github.workspace }}/lute/config/config.yml
        echo "DATAPATH: ${{ github.workspace }}/data" >> ${{ github.workspace }}/lute/config/config.yml
        echo "DBNAME: test_lute.db" >> ${{ github.workspace }}/lute/config/config.yml
        ls ${{ github.workspace }}
        cat ${{ github.workspace }}/lute/config/config.yml

    - name: Test
      run: |
        set -e
        # Have to explicitly set MECAB_PATH for natto-py.
        export MECAB_PATH=/lib/x86_64-linux-gnu/libmecab.so.2
        pytest

    - name: Acceptance testing
      uses: nick-fields/retry@v3
      with:
        max_attempts: 3
        timeout_minutes: 30
        command: |
          # handle errors manually, so that the retry works.
          exit_code=0
          # Have to explicitly set MECAB_PATH for natto-py.
          export MECAB_PATH=/lib/x86_64-linux-gnu/libmecab.so.2
          inv accept --show --verbose || exit_code=$?
          if [ "$exit_code" -ne 0 ]; then
              echo "Command failed with exit code $exit_code, retrying..."
          fi
          exit $exit_code

    # Playwright tests were hanging far too often on github ci,
    # but not consistently.  No idea what was going wrong.
    # TODO github ci: fix playwright hanging.
    - name: Playwright install
      run: playwright install

    - name: Playwright smoke test
      run: |
        set -x
        set -e
        inv playwright

    - name: Check flit package
      run: |
        mkdir ${{ github.workspace }}/../lute_flit
        cd ${{ github.workspace }}/../lute_flit
        flit -f ${{ github.workspace }}/pyproject.toml install

        python -m lute.main &   # Start in background process
        sleep 2

        # Verify with utils script back in the workspace
        # (verify.py is not included in flit install)
        pushd ${{ github.workspace }}
        python -m utils.verify 5001
        popd

        pkill -f "python -m lute.main"   # Kill that process.


  # Lute should still work if the user doesn't have mecab installed.
  no-mecab-check:
    runs-on: ubuntu-latest
    timeout-minutes: 10

    strategy:
      matrix:
        # Only checking early and late versions.
        python_version: [ '3.8', '3.11' ]

    steps:

    - uses: actions/checkout@v4
      with:
        submodules: true

    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python_version }}
        cache: 'pip' # caching pip dependencies

    - run: pip install -r requirements-dev.txt

    - name: Setup config
      run: |
        mkdir ${{ github.workspace }}/data
        echo "ENV: dev" > ${{ github.workspace }}/lute/config/config.yml
        echo "DATAPATH: ${{ github.workspace }}/data" >> ${{ github.workspace }}/lute/config/config.yml
        echo "DBNAME: test_lute.db" >> ${{ github.workspace }}/lute/config/config.yml
        ls ${{ github.workspace }}
        cat ${{ github.workspace }}/lute/config/config.yml

    # Run one particular sanity check.
    #
    # Note this isn't _completely_ valid because the test loads
    # only supported language stories, whereas a prod release
    # comes with _all_ stories pre-loaded and the invalid ones
    # are deleted ...
    - name: Smoke test no mecab
      run: inv accept -s -k disabled_data_is_hidden


  # Run all plugin tests.
  #
  # For each plugin:
  #   - install Lute requirements
  #   - install plugin reqs
  #   - run tests.
  #
  # The Lute requirements are installed first b/c the plugins may come
  # with their own conflicting requirements.  Doing a full req install
  # will (hopefully) uncover conflicts.
  plugins:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    strategy:
      matrix:
        python_version: [ '3.8', '3.9', '3.10', '3.11' ]

    steps:

    - uses: actions/checkout@v4
      with:
        submodules: true

    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python_version }}
        cache: 'pip' # caching pip dependencies

    # Plugins likely won't need this config file, but just in case ...
    - name: Setup config
      run: |
        mkdir ${{ github.workspace }}/data
        echo "ENV: dev" > ${{ github.workspace }}/lute/config/config.yml
        echo "DATAPATH: ${{ github.workspace }}/data" >> ${{ github.workspace }}/lute/config/config.yml
        echo "DBNAME: test_lute.db" >> ${{ github.workspace }}/lute/config/config.yml
        ls ${{ github.workspace }}
        cat ${{ github.workspace }}/lute/config/config.yml

    - name: test all plugins
      run: |
        for plugin in $(ls plugins); do
          # Lute reqs, such as pytest.
          pip install -r requirements-dev.txt

          # Lute itself, via toml, so that it can be found
          # by each plugin's own "pip install ."
          pip install .

          pushd plugins/$plugin
            pip install .
            # Note for future: some plugins may have extra reqs not covered by pip
            # (e.g. mecab uses apt-get and exports etc).  Idea for future: plugin
            # could have a .github folder as well with additional setup scripts.

            pytest tests

            # pip uninstall $plugin -y
            # NOTE: Not bothering to do an uninstall!
            # if multiple plugins have different/clashing version requirements,
            # perhaps it is best to run into problems in ci.
            # This may ultimately come back to haunt me, but it will do for now.
          popd
        done


  code-quality:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    strategy:
      matrix:
        python_version: [ '3.11' ]

    steps:

    - uses: actions/checkout@v4
      with:
        submodules: true


    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python_version }}
        cache: 'pip' # caching pip dependencies

    - run: pip install -r requirements-dev.txt

    - name: Lint
      run: inv lint

    - name: Coding style
      run: |
        black --check .


  # Test build docker container and try running.
  # Slightly wasteful re-setup of node.
  docker-build:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    needs: base-ci
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: actions/setup-python@v4
      with:
        python-version: 3.11
        cache: 'pip' # caching pip dependencies
    - run: pip install -r requirements-dev.txt
    - name: Test docker build
      run: |
        docker build -f docker/Dockerfile --build-arg INSTALL_EVERYTHING=false -t lute3 .
        # Run container in the background, and check.
        docker run -d -p 5001:5001 -v ./my_data:/lute_data -v ./my_backups:/lute_backup --name my-lute lute3:latest
        sleep 10  # Give it a moment to start.
        python -m utils.verify 5001
        docker stop my-lute
        docker rm my-lute


  # Ensure that basic things work correctly on Windows,
  # particularly PlatformDirs package.
  windows-ci:

    runs-on: windows-latest
    timeout-minutes: 30

    strategy:
      matrix:
        python_version: [ '3.8', '3.9', '3.10', '3.11' ]

    steps:

    - uses: actions/checkout@v4
      with:
        submodules: true


    - uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python_version }}
        cache: 'pip' # caching pip dependencies

    - name: Setup config
      run: |
        mkdir ${{ github.workspace }}\data
        echo ENV: dev > ${{ github.workspace }}\lute\config\config.yml
        echo DATAPATH: ${{ github.workspace }}\data >> ${{ github.workspace }}\lute\config\config.yml
        echo DBNAME: test_lute.db >> ${{ github.workspace }}\lute\config\config.yml
      shell: cmd

    - name: Replace config slashes.
      run: |
        $content = [System.IO.File]::ReadAllText("${{ github.workspace }}\lute\config\config.yml").Replace('\','/')
        [System.IO.File]::WriteAllText("${{ github.workspace }}\lute\config\config.yml", $content)
        Get-Content -Path "${{ github.workspace }}\lute\config\config.yml"

    - run: pip install -r requirements-dev.txt

    # Can't get playwright to run reliably on Windows.
    # - name: Playwright install
    #   run: playwright install
    # - name: Playwright smoke test
    #   run: inv playwright || exit /b

    # Now having problems with tests not working on windows ...
    # getting failure message:
    # javascript error: clear_datatable_state is not defined
    #
    # The above message is called from lute_test_client to clear book
    # datatables state.  This _used_ to work (e.g. in v3.3.0), and
    # I can't track it down at the moment!!!!!!
    #
    # TODO ci: RESTORE AT LEAST ONE SANITY CHECK TEST ON WINDOWS.
    #
    # Run specific sanity check.
    # Old tests no longer run -- datatables may have timing issues on Windows tests,
    # tests were far too flaky.
    # inv accept -s -k test_unsupported_language_not_shown || exit /b
    # inv accept -s -k import_a_valid_term_file || exit /b
    # - name: Smoke tests
    #   run: |
    #     inv accept -s -k test_updating_term_status_updates_the_reading_frame || exit /b

    - name: Remove config to force using prod config
      run: del ${{ github.workspace }}\lute\config\config.yml
      shell: cmd

    - name: Install prod-like flit package
      run: |
        mkdir ${{ github.workspace }}\..\lute_flit
        cd ${{ github.workspace }}\..\lute_flit
        flit -f ${{ github.workspace }}\pyproject.toml install
      shell: cmd

    - name: Run Lute application
      run: |
        echo "python -m lute.main" > run-server.bat
        cmd /c "START /b run-server.bat"

    - name: Wait for application to start
      run: Start-Sleep -Seconds 10

    - name: Verify flit install
      run: |
        cd ${{ github.workspace }}
        python -m utils.verify 5001