test-platforms.yml

# Run kickstart tests in a PR triggered by a "/test-platforms" command from an organization member
#
# /test-platforms - tests all tests affected by the PR or smoke tests if there is no such
# Or specify tests by combination of test names and/or options:
# /test-platforms --testtype TESTTYPE --skip-testtypes TYPE[,TYPE..] TEST1 TEST2
name: test-platforms
on:
  issue_comment:
    types: [created]

permissions:
  contents: read
  statuses: write

jobs:
  pr-info:
    if: startsWith(github.event.comment.body, '/test-platforms')
    runs-on: ubuntu-latest
    steps:
      - name: Query comment author repository permissions
        uses: octokit/request-action@v2.x
        id: user_permission
        with:
          route: GET /repos/${{ github.repository }}/collaborators/${{ github.event.sender.login }}/permission
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # restrict running of tests to users with admin or write permission for the repository
      # see https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#get-repository-permissions-for-a-user
      # store output if user is allowed in allowed_user job output so it has to be checked in downstream job
      - name: Check if user does have correct permissions
        if: contains('admin write', fromJson(steps.user_permission.outputs.data).permission)
        id: check_user_perm
        run: |
          echo "User '${{ github.event.sender.login }}' has permission '${{ fromJson(steps.user_permission.outputs.data).permission }}' allowed values: 'admin', 'write'"
          echo "allowed_user=true" >> $GITHUB_OUTPUT

      - name: Get information for pull request
        uses: octokit/request-action@v2.x
        id: pr_api
        with:
          route: GET /repos/${{ github.repository }}/pulls/${{ github.event.issue.number }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Parse launch arguments
        id: parse_launch_args
        # Do not use comment body directly in the shell command to avoid possible code injection.
        env:
          BODY: ${{ github.event.comment.body }}
        run: |
          # extract first line and cut out the "/kickstart-tests" first word
          LAUNCH_ARGS=$(echo "$BODY" | sed -n '1 s/^[^ ]* *//p' | sed 's/[[:space:]]*$//')
          echo "launch arguments are: $LAUNCH_ARGS"
          echo "launch_args=${LAUNCH_ARGS}" >> $GITHUB_OUTPUT

    outputs:
      allowed_user: ${{ steps.check_user_perm.outputs.allowed_user }}
      base_ref: ${{ fromJson(steps.pr_api.outputs.data).base.ref }}
      sha: ${{ fromJson(steps.pr_api.outputs.data).head.sha }}
      launch_args: ${{ steps.parse_launch_args.outputs.launch_args }}


  platform:
    needs: pr-info
    if: needs.pr-info.outputs.allowed_user == 'true' && ! contains(github.event.comment.body, '--waive')
    name: Run tests on the platform
    runs-on: [self-hosted, kstest-test]
    env:
      TARGET_BRANCH: ${{ needs.pr-info.outputs.base_ref }}
      TEST_JOBS: 16
      GITHUB_TOKEN: /home/github/github-token
    strategy:
      matrix:
        #platform: [daily-iso, rawhide, rhel8, rhel9]
        #platform: [daily-iso]
        platform: [rawhide]
      fail-fast: false

    steps:
      # self-hosted runners don't do this automatically; also useful to keep stuff around for debugging
      # need to run sudo as the launch script and the container create root/other user owned files
      - name: Clean up previous run
        run: |
          sudo podman ps -q --all --filter='ancestor=kstest-runner' | xargs -tr sudo podman rm -f
          sudo podman volume rm --all || true
          sudo rm -rf * .git

      - name: Clone repository
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.pr-info.outputs.sha }}
          path: kickstart-tests

      - name: Check out kickstart-tests
        uses: actions/checkout@v4
        with:
          #TODO change
          #repository: rhinstaller/kickstart-tests
          repository: rvykydal/kickstart-tests
          path: kickstart-tests

      - name: Generate test cases
        working-directory: ./kickstart-tests
        run: scripts/generate-testcases.py -t ./testlib/test_cases/kstest-template.tc.yaml.j2 . -o ./testlib/test_cases

      - name: Clone Permian repository
        uses: actions/checkout@v4
        with:
          repository: rhinstaller/permian
          path: permian
          ref: main

      - name: Clone tplib repository
        uses: actions/checkout@v4
        with:
          repository: rhinstaller/tplib
          path: tplib

      - name: Ensure http proxy is running
        run: sudo containers/squid.sh start
        working-directory: ./kickstart-tests

      - name: Update container images used here
        run: |
          sudo podman pull quay.io/rhinstaller/kstest-runner:latest

      - name: Get changed tests
        working-directory: ./kickstart-tests
        id: get_changed_tests
        run: |
          git fetch origin
          # TODO: share with scripts/run-travis.sh
          CHANGED_TESTS=$(git diff --name-only origin/$TARGET_BRANCH ${{ needs.pr-info.outputs.sha }} -- *.ks.in $(find -maxdepth 1 -name '*.sh' -perm -u+x) | sed 's/\.ks\.in$//; s/\.sh$//' | sort -u | tr '\n' ' ')
          echo "changed_tests=${CHANGED_TESTS}" >> $GITHUB_OUTPUT

      - name: Get skipped tests for platform ${{ matrix.platform }}
        id: get_platform_skiptest
        working-directory: ./kickstart-tests
        run: |
          set -eux

          if [ ${{ matrix.platform }} == "daily-iso" ]; then
            echo "skip_tests=skip-on-fedora" >> $GITHUB_OUTPUT
          elif [ ${{ matrix.platform }} == "rawhide" ]; then
            echo "skip_tests=skip-on-fedora" >> $GITHUB_OUTPUT
          elif [ ${{ matrix.platform }} == "rhel8" ]; then
            echo "skip_tests=skip-on-rhel,skip-on-rhel-8" >> $GITHUB_OUTPUT
          elif [ ${{ matrix.platform }} == "rhel9" ]; then
            echo "skip_tests=skip-on-rhel,skip-on-rhel-9" >> $GITHUB_OUTPUT
          else
            echo "Platform is not supported by kickstart tests yet!"
            exit 1
          fi

      - name: Generate test selection
        id: generate_query
        working-directory: ./kickstart-tests
        run: |
          set -eux

          if [ -n "${{ needs.pr-info.outputs.launch_args" }} ]; then
            echo "Running tests required by the comment explicitly."
            LAUNCH_ARGS = "${{ needs.pr-info.outputs.launch_args }}"
          elif [ -n "${{ steps.get_changed_tests.outputs.changed_tests }}" ]; then
            echo "Running tests affected by changes."
            LAUNCH_ARGS = "${{ steps.get_changed_tests.outputs.changed_tests }}"
          else
            echo "Running smoke tests."
            LAUNCH_ARGS = "--testtype smoke"
          fi

          source ./containers/runner/skip-testtypes
          PERMIAN_QUERY=$(scripts/generate-permian-query.py \
             --skip-testtypes $SKIP_TESTTYPES_ANACONDA_PR \
             --skip-testtypes ${{ steps.get_platform_skiptest.outputs.skip_tests }} \
             ${LAUNCH_ARGS} )
          if [ $? == 0 ]; then
            echo "query=$PERMIAN_QUERY" >> $GITHUB_OUTPUT
          else
            echo "Parsing of the request arguments failed"
            exit 1
          fi

          #      # Fetch boot.iso and configiure its local location
          #      - name: Fetch boot.iso if available for platform ${{ matrix.platform }}
          #        id: boot_iso_for_platform
          #        run: |
          #          set -eux
          #          BOOT_ISO_PATH="${{ github.workspace }}/${{ matrix.platform }}.boot.iso"
          #          BOOT_ISO_URL="file://$BOOT_ISO_PATH"
          #          if [ "${{ matrix.platform }}" == "daily-iso" ]; then
          #            ${{ github.workspace }}/kickstart-tests/containers/runner/fetch_daily_iso.sh $GITHUB_TOKEN $BOOT_ISO_PATH
          #            echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
          #          elif [ "${{ matrix.platform }}" == "rawhide" ]; then
          #            curl -L https://download.fedoraproject.org/pub/fedora/linux/development/rawhide/Server/x86_64/os/images/boot.iso --output $BOOT_ISO_PATH
          #            echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
          #          else
          #            echo "Boot.iso for ${{ matrix.platform }} can't be fetched."
          #            echo "boot_iso=" >> $GITHUB_OUTPUT
          #          fi
          #
          #      # Configure location of installation repositories for the platform
          #      # Also default boot.iso is defined by the value of urls.installation_tree
          #      # of kstestParams event structure.
          #      - name: Set installation_tree for the platform
          #        id: set_installation_urls
          #        run: |
          #          set -eux
          #          if [ "${{ matrix.platform }}" == "rhel8" ]; then
          #            echo "installation_tree=http://download.eng.bos.redhat.com/rhel-8/nightly/RHEL-8/latest-RHEL-8.10.0/compose/BaseOS/x86_64/os" >> $GITHUB_OUTPUT
          #            echo "modular_url=http://download.eng.bos.redhat.com/rhel-8/nightly/RHEL-8/latest-RHEL-8.10.0/compose/AppStream/x86_64/os" >> $GITHUB_OUTPUT
          #          elif [ "${{ matrix.platform }}" == "rhel9" ]; then
          #            echo "installation_tree=http://download.eng.bos.redhat.com/rhel-9/nightly/RHEL-9/latest-RHEL-9.4.0/compose/BaseOS/x86_64/os" >> $GITHUB_OUTPUT
          #            echo "modular_url=http://download.eng.bos.redhat.com/rhel-9/nightly/RHEL-9/latest-RHEL-9.4.0/compose/AppStream/x86_64/os" >> $GITHUB_OUTPUT
          #          else
          #            echo "Installation tree location for ${{ matrix.platform }} not configured"
          #            if [ -z "${{ steps.boot_iso_for_platform.outputs.boot_iso }}" ]; then
          #              echo "No boot.iso source is defined"
          #              exit 2
          #            fi
          #            echo "installation_tree=" >> $GITHUB_OUTPUT
          #            echo "modular_url=" >> $GITHUB_OUTPUT
          #          fi
          #
          #      - name: Create Permian settings file
          #        working-directory: ./permian
          #        run: |
          #          cat <<EOF > settings.ini
          #          [kickstart_test]
          #          kstest_local_repo=${{ github.workspace }}/kickstart-tests
          #          [library]
          #          directPath=${{ github.workspace }}/kickstart-tests/testlib
          #          EOF
          #
          #      - name: Run kickstart tests in container
          #        working-directory: ./permian
          #        run: |
          #          sudo --preserve-env=TEST_JOBS \
          #          PYTHONPATH=${PYTHONPATH:-}:${{ github.workspace }}/tplib \
          #          ./run_subset --debug-log permian.log \
          #            --settings settings.ini \
          #            --override workflows.dry_run=True \
          #            --testcase-query '${{ steps.generate_query.outputs.query }}' \
          #            run_event '{
          #              "type":"everything",
          #              "everything_testplan":{
          #                "configurations":[{"architecture":"x86_64"}],
          #                "point_person":"rvykydal@redhat.com"
          #              },
          #              ${{ steps.boot_iso_from_platform.outputs.boot_iso }}
          #              "kstestParams":{
          #                "platform":"${{ steps.platform_from_scenario.outputs.platform }}",
          #                "urls":{
          #                  "x86_64":{
          #                    "installation_tree":"${{ steps.set_installation_urls.outputs.installation_tree }}",
          #                    "modular_url":"${{ steps.set_installation_urls.outputs.modular_url }}"
          #                  }
          #                }
          #              }
          #            }'
          #
          #          # Permian hides the exit code of launcher, so error out this step manually based on logs
          #          rc=$( awk '/Runner return code: /{ print $4 }' permian.log)
          #          if [ -n "$rc" ]; then
          #            exit $rc
          #          else
          #            exit 111
          #          fi
          #
          #      - name: Collect anaconda logs
          #        if: always()
          #        uses: actions/upload-artifact@v3
          #        with:
          #          name: 'logs-${{ matrix.scenario }}'
          #          # skip the /anaconda subdirectories, too large
          #          path: |
          #            kickstart-tests/data/logs/kstest*.log
          #            kickstart-tests/data/logs/kstest.log.json
          #            kickstart-tests/data/logs/kstest-*/*.log
          #            kickstart-tests/data/logs/kstest-*/anaconda/lorax-packages.log
          #
          #      - name: Collect Permian logs
          #        if: always()
          #        uses: actions/upload-artifact@v3
          #        with:
          #          name: 'logs-permian-${{ matrix.scenario }}'
          #          path: |
          #            permian/permian.log
          #
          #    # TODO: use supplied arguments (overriding)
          #    # TODO: timeouts?
          #    # TODO: implement waive
          #    # TODO: solve empty changed tests (smoke?)
          #    # TODO: implement progress from anaconda repo
          #    # TODO?: installation_tree from defaults file ?