run-tests.yml

name: 🧪 QGIS tests

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

on:
  push:
    branches:
      - master
      - release-**
      - queued_ltr_backports
    paths:
    - 'src/**'
    - 'external/**'
    - 'python/**'
    - 'tests/**'
    - 'CMakeLists.txt'
    - '.github/workflows/run-tests.yml'
    - '.docker/**'
    - '.ci/**'
  pull_request:
    branches:
      - master
      - release-**
      - queued_ltr_backports
    # paths cannot be filtered on this workflow on pull request as it is a required one in the branch protection
    # feature request and hacks: https://github.community/t/feature-request-conditional-required-checks/16761

permissions:
  contents: read

jobs:
  build:
    env:
      QGIS_WORKSPACE: ${{ github.workspace }} # used in docker compose
      RUN_FLAKY_TESTS: ${{ contains( github.event.pull_request.labels.*.name, 'run flaky tests') }}

    runs-on: ubuntu-latest
    continue-on-error: ${{ matrix.experimental }}

    strategy:
      matrix:
        include:
          - distro-version: '24.04'
            qt-version: 5
            run-tests: true
            with-qt6: OFF
            with-qt5: ON
            with-3d: ON
            with-quick: ON
            with-clazy: ON
            with-grass7: OFF
            with-grass8: OFF
            with-webengine: OFF
            with-pdf4qt: OFF
            with-compile-commands: ON
            # LD_PRELOAD: /lib/x86_64-linux-gnu/libSegFault.so
            experimental: false
            unity-builds: ON

          - distro-version: '39'
            qt-version: 6
            run-tests: true
            with-qt6: ON
            with-qt5: OFF
            with-3d: ON
            with-quick: ON
            with-clazy: OFF
            with-grass7: OFF
            with-grass8: ON
            with-webengine: ON
            with-pdf4qt: ON
            with-compile-commands: OFF
            LD_PRELOAD: ''
            experimental: false
            unity-builds: ON


      fail-fast: false

    outputs:
      compile_outcome: ${{ steps.compile.outcome }}
      tests_failing: ${{ steps.tests.outputs.TESTS_FAILING }}
      cdash_url: ${{ steps.tests.outputs.CDASH_URL }}
      runners_outcome: ${{ steps.runners.outcome }}

    steps:

      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          large-packages: false
          docker-images: false
          swap-storage: true

      - name: Checkout
        uses: actions/checkout@v4

      - name: Set vars
        env:
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          GITHUB_REF: ${{ github.ref }}
          GITHUB_PR_NUMBER: ${{github.event.number}}
        run: |
          # Be aware that these instructions are duplicated in run-tests job
          DOCKER_TAG=$(echo $( [[ ${GITHUB_EVENT_NAME} == pull_request ]] && echo ${GITHUB_BASE_REF} || echo ${GITHUB_REF##*/} ) | sed 's/^master$/latest/')
          CTEST_BUILD_NAME=$( [[ ${GITHUB_EVENT_NAME} == pull_request ]] && echo "PR${GITHUB_PR_NUMBER}" || echo ${GITHUB_REF##*/} )"_${GITHUB_SHA}"
          echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV
          echo "CTEST_BUILD_NAME=${CTEST_BUILD_NAME}" >> $GITHUB_ENV
          echo "QT_VERSION=${{ matrix.qt-version }}" >> $GITHUB_ENV

      - name: Print vars
        run: |
          echo DOCKER_TAG: ${DOCKER_TAG}
          echo CTEST_BUILD_NAME: ${CTEST_BUILD_NAME}
          echo QT_VERSION: ${QT_VERSION}

      - name: Login to Docker Hub
        if: ${{ github.event_name == 'push' && github.repository == 'qgis/QGIS' }}
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Build Docker Container with Build Environment
        id: docker-build
        uses: docker/build-push-action@v6
        with:
          context: .
          file: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile
          tags: qgis/qgis3-build-deps-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}:${{ github.event.pull_request.base.ref || github.ref_name }}
          push: ${{ github.event_name == 'push' && github.repository == 'qgis/QGIS' }}
          pull: true
          build-args:
            DISTRO_VERSION=${{ matrix.distro-version }}

      - name: Tag image
        run: docker tag ${{ steps.docker-build.outputs.imageid }} qgis3-build-deps

      - name: Restore build cache
        uses: actions/cache/restore@v4
        with:
          path: /home/runner/QGIS/.ccache
          key: build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.event.pull_request.base.ref || github.ref_name }}
          restore-keys: |
            build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-master

      - name: Compile QGIS
        id: compile
        run: |
          docker run -t --name qgis_container \
                     -v $(pwd):/root/QGIS \
                     -v /home/runner/QGIS/.ccache:/root/.ccache \
                     --env-file .docker/docker-variables.env \
                     --env CCACHE_DIR=/root/.ccache \
                     --env PUSH_TO_CDASH=true \
                     --env WITH_QT5=${{ matrix.with-qt5 }} \
                     --env BUILD_WITH_QT6=${{ matrix.with-qt6 }} \
                     --env WITH_QUICK=${{ matrix.with-quick }} \
                     --env WITH_3D=${{ matrix.with-3d }} \
                     --env WITH_GRASS7=${{ matrix.with-grass7 }} \
                     --env WITH_GRASS8=${{ matrix.with-grass8 }} \
                     --env WITH_QTWEBENGINE=${{ matrix.with-webengine }} \
                     --env WITH_PDF4QT=${{ matrix.with-pdf4qt }} \
                     --env LD_PRELOAD=${{ matrix.LD_PRELOAD }} \
                     --env WITH_CLAZY=${{ matrix.with-clazy }} \
                     --env WITH_COMPILE_COMMANDS=${{ matrix.with-compile-commands }} \
                     --env ENABLE_UNITY_BUILDS=${{ matrix.unity-builds }} \
                     qgis3-build-deps \
                     /root/QGIS/.docker/docker-qgis-build.sh

      - name: Save build cache for push only
        uses: actions/cache/save@v4
        if: ${{ github.event_name == 'push' }}
        with:
          path: /home/runner/QGIS/.ccache
          key: build-ccache-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}-${{ github.ref_name }}-${{ github.run_id }}

      - name: Push artifact
        id: push_artifact
        if: ${{ matrix.run-tests }}
        run: tar --exclude='*.o' -cvzf build.tgz build

      - uses: actions/upload-artifact@v4
        if: ${{ matrix.run-tests }}
        with:
          name: build-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}.tgz
          path: ./build.tgz
          retention-days: 1

#      - name: Test QGIS runners
#        id: runners
#        if: ${{ matrix.distro-version == env.DEFAULT_DISTRO_VERSION }}
#        run: |
#          docker run -d --name qgis-testing-environment \
#                     -v $(pwd):/root/QGIS \
#                     -v $(pwd)/tests/src/python:/tests_directory \
#                     -v $(pwd)/.docker/qgis_resources/test_runner:/usr/bin/test_runner \
#                     -v $(pwd)/.docker/qgis_resources/supervisor:/etc/supervisor \
#                     -e QGIS_BUILD_PATH=/root/QGIS/build/output/bin/qgis \
#                     -e TEST_RUNNER_PATH=/usr/bin/test_runner/qgis_testrunner.py \
#                     -e DISPLAY=:99 \
#                     qgis_image \
#                     /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
#          # Wait for xvfb to finish starting
#          printf "Waiting for the docker...🐳..."
#          sleep 10
#          echo " done 🥩"
#
#          declare -A testrunners
#          # Passing cases:
#          testrunners["test_testrunner.run_passing"]=0
#          testrunners["test_testrunner.run_skipped_and_passing"]=0
#          # Failing cases:
#          testrunners["test_testrunner"]=1
#          testrunners["test_testrunner.run_all"]=1
#          testrunners["test_testrunner.run_failing"]=1
#          set +e # do not exit on error
#          # Run tests in the docker
#          for i in "${!testrunners[@]}"
#          do
#            echo "::group::docker_test_runner_${i}"
#            echo "test ${i}..."
#            docker exec -i qgis-testing-environment sh -c "cd /tests_directory && /usr/bin/test_runner/qgis_testrunner.sh ${i}"
#            [[ $? -eq "${testrunners[$i]}" ]] && echo "success" || exit 1
#            echo "::endgroup::"
#          done
#          set -e # switch back
#          docker stop qgis-testing-environment

  run-tests:
    name: Run tests
    env:
      QGIS_WORKSPACE: ${{ github.workspace }} # used in docker compose

    runs-on: ubuntu-latest

    needs: build
    if: always()

    strategy:
      matrix:
        qt-version: [5, 6]
        test-batch: [ALL_BUT_PROVIDERS, POSTGRES, HANA, SQLSERVER]

        include:
          - qt-version: 5
            distro-version: 24.04
            docker-target: binary-only

          - qt-version: 6
            distro-version: 39
            docker-target: binary-only

          - qt-version: 5
            distro-version: 24.04
            test-batch: ORACLE
            docker-target: binary-for-oracle

        exclude:
          - qt-version: 6
            test-batch: HANA

          - qt-version: 6
            test-batch: POSTGRES

          - qt-version: 6
            test-batch: SQLSERVER

      fail-fast: false

    steps:

      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          large-packages: false
          docker-images: false
          swap-storage: true

      - name: Checkout
        uses: actions/checkout@v4

      - name: Set vars
        env:
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          GITHUB_REF: ${{ github.ref }}
          GITHUB_PR_NUMBER: ${{github.event.number}}
        run: |
          # Be aware that these instructions are duplicated in build job
          CTEST_BUILD_NAME=$( [[ ${GITHUB_EVENT_NAME} == pull_request ]] && echo "PR${GITHUB_PR_NUMBER}" || echo ${GITHUB_REF##*/} )"_${GITHUB_SHA}_${{ matrix.test-batch }}"
          echo "CTEST_BUILD_NAME=${CTEST_BUILD_NAME}" >> $GITHUB_ENV
          echo "QT_VERSION=${{ matrix.qt-version }}" >> $GITHUB_ENV

      - name: Print vars
        run: |
          echo CTEST_BUILD_NAME: ${CTEST_BUILD_NAME}

      - name: Login to Docker Hub
        if: ${{ github.event_name == 'push' && github.repository == 'qgis/QGIS' }}
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Build Docker Container with Testing Environment
        id: docker-build
        uses: docker/build-push-action@v6
        with:
          context: .
          file: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile
          tags: qgis/qgis3-qt${{ matrix.qt-version }}-build-deps-bin-only:${{ github.event.pull_request.base.ref || github.ref_name }}
          push: ${{ github.event_name == 'push' && github.repository == 'qgis/QGIS' }}
          pull: true
          target: ${{ matrix.docker-target }}
          build-args:
            DISTRO_VERSION=${{ matrix.distro-version }}

      - name: Tag image
        run: docker tag ${{ steps.docker-build.outputs.imageid }} qgis3-build-deps-binary-image

      - name: Print disk space
        run: |
          echo "DOCKER IMAGES"
          docker images
          echo "DF -H"
          sudo df -h

      - name: Download build artifact
        uses: actions/download-artifact@v4
        with:
          name: build-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}.tgz
          path: .

      - name: Extract build artifact
        run: |
          tar xvzf build.tgz
          rm -Rf build.tgz

      - name: Print disk space
        run: |
          echo "DF -H"
          sudo df -h

      - name: Run tests
        id: tests
        env:
          TEST_BATCH: ${{matrix.test-batch}}
          QGIS_COMMON_GIT_DIR: ${{ github.workspace }}
          GITHUB_SHA: ${{ github.sha }}
        run: |
          DOCKERFILE=$( ( [[ ${{ matrix.test-batch }} == "ORACLE"   ]] && echo "docker-compose-testing-oracle.yml" ) \
                     || ( [[ ${{ matrix.test-batch }} == "POSTGRES" ]] && echo "docker-compose-testing-postgres.yml" ) \
                     || ( [[ ${{ matrix.test-batch }} == "SQLSERVER" ]] && echo "docker-compose-testing-mssql.yml" ) \
                     || echo "docker-compose-testing.yml" )
          [[ ${{ matrix.test-batch }} == "ORACLE" ]] && sudo rm -rf /usr/share/dotnet/sdk
          echo "TEST_BATCH=$TEST_BATCH"
          echo "DOCKERFILE=$DOCKERFILE"
          mkdir -p /tmp/webdav_tests && chmod 777 /tmp/webdav_tests
          mkdir -p /tmp/minio_tests/test-bucket && chmod -R 777 /tmp/minio_tests
          docker compose -f .docker/$DOCKERFILE run -e GITHUB_SHA=$GITHUB_SHA qgis-deps /root/QGIS/.docker/docker-qgis-test.sh $TEST_BATCH

      - name: Fix permissions on test report
        if: ${{ failure() }}
        run: |
          sudo chmod -R 777 qgis_test_report

      - name: Dump report contents
        if: ${{ failure() }}
        run: |
          MD_REPORT_FILE="qgis_test_report/summary.md"; [ -f "$MD_REPORT_FILE" ] && cat "$MD_REPORT_FILE" || true

      - name: Save PR number to test report
        if: ${{ failure() }}
        run: |
          echo ${{ github.event.number }} | tee qgis_test_report/pr_number
          echo ${{ github.event.pull_request.head.sha }} | tee qgis_test_report/git_commit

      - name: Archive test results report
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: test-results-qt${{ matrix.qt-version }}
          path: qgis_test_report

  clang-tidy:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    needs: build

    strategy:
      matrix:
        include:
          - distro-version: '24.04'
            qt-version: 5

    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: true
          large-packages: false
          docker-images: false
          swap-storage: true

      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Login to Docker Hub
        if: ${{ github.event_name == 'push' && github.repository == 'qgis/QGIS' }}
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Build Docker Container with Testing Environment
        id: docker-build
        uses: docker/build-push-action@v6
        with:
          context: .
          file: .docker/qgis3-qt${{ matrix.qt-version }}-build-deps.dockerfile
          tags: qgis/qgis3-qt${{ matrix.qt-version }}-build-deps-bin-only:${{ github.event.pull_request.base.ref || github.ref_name }}
          push: ${{ github.event_name == 'push' && github.repository == 'qgis/QGIS' }}
          pull: true
          target: ${{ matrix.docker-target }}
          build-args:
            DISTRO_VERSION=${{ matrix.distro-version }}

      - name: Tag image
        run: docker tag ${{ steps.docker-build.outputs.imageid }} qgis3-build-deps-binary-image

      - name: Download build artifact
        uses: actions/download-artifact@v4
        with:
          name: build-${{ matrix.distro-version }}-qt${{ matrix.qt-version }}.tgz
          path: .

      - name: Extract build artifact
        run: |
          tar xvzf build.tgz
          rm -Rf build.tgz

      - name: Run Clang-Tidy
        run: |
          docker run -t --name qgis_container \
          -v $(pwd):/root/QGIS \
          -v /home/runner/QGIS/.ccache:/root/.ccache \
          --env-file .docker/docker-variables.env \
          qgis3-build-deps-binary-image \
          /root/QGIS/.docker/docker-qgis-clangtidy.sh