test-platforms.yml

# Run kickstart tests in a PR triggered by a "/test-platforms" command from an organization member
#
# /test-platforms - tests all tests affected by the PR or smoke tests if there is no such
# Or specify tests by combination of test names and/or options:
# /test-platforms --testtype TESTTYPE --skip-testtypes TYPE[,TYPE..] TEST1 TEST2
name: test-platforms
on:
  issue_comment:
    types: [created]

permissions:
  contents: read
  statuses: write

jobs:
  pr-info:
    if: startsWith(github.event.comment.body, '/test-platforms')
    runs-on: ubuntu-latest
    steps:
      - name: Query comment author repository permissions
        uses: octokit/request-action@v2.x
        id: user_permission
        with:
          route: GET /repos/${{ github.repository }}/collaborators/${{ github.event.sender.login }}/permission
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # restrict running of tests to users with admin or write permission for the repository
      # see https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#get-repository-permissions-for-a-user
      # store output if user is allowed in allowed_user job output so it has to be checked in downstream job
      - name: Check if user does have correct permissions
        if: contains('admin write', fromJson(steps.user_permission.outputs.data).permission)
        id: check_user_perm
        run: |
          echo "User '${{ github.event.sender.login }}' has permission '${{ fromJson(steps.user_permission.outputs.data).permission }}' allowed values: 'admin', 'write'"
          echo "allowed_user=true" >> $GITHUB_OUTPUT

      - name: Get information for pull request
        uses: octokit/request-action@v2.x
        id: pr_api
        with:
          route: GET /repos/${{ github.repository }}/pulls/${{ github.event.issue.number }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Parse launch arguments
        id: parse_launch_args
        # Do not use comment body directly in the shell command to avoid possible code injection.
        env:
          BODY: ${{ github.event.comment.body }}
        run: |
          # extract first line and cut out the "/kickstart-tests" first word
          LAUNCH_ARGS=$(echo "$BODY" | sed -n '1 s/^[^ ]* *//p' | sed 's/[[:space:]]*$//')
          echo "launch arguments are: $LAUNCH_ARGS"
          echo "launch_args=${LAUNCH_ARGS}" >> $GITHUB_OUTPUT

    outputs:
      allowed_user: ${{ steps.check_user_perm.outputs.allowed_user }}
      base_ref: ${{ fromJson(steps.pr_api.outputs.data).base.ref }}
      sha: ${{ fromJson(steps.pr_api.outputs.data).head.sha }}
      launch_args: ${{ steps.parse_launch_args.outputs.launch_args }}


  platform:
    needs: pr-info
    if: needs.pr-info.outputs.allowed_user == 'true'
    name: Run tests on the platform
    runs-on: [self-hosted, kstest-test]
    env:
      STATUS_NAME: test-platforms
      TARGET_BRANCH: ${{ needs.pr-info.outputs.base_ref }}
      TEST_JOBS: 16
      GITHUB_TOKEN: /home/github/github-token
    strategy:
      matrix:
        platform: [rhel8]
        #platform: [daily-iso, rawhide, rhel8, rhel9]
      fail-fast: false

    steps:
      # self-hosted runners don't do this automatically; also useful to keep stuff around for debugging
      # need to run sudo as the launch script and the container create root/other user owned files
      - name: Clean up previous run
        run: |
          sudo podman ps -q --all --filter='ancestor=kstest-runner' | xargs -tr sudo podman rm -f
          sudo podman volume rm --all || true
          sudo rm -rf * .git

      - name: Clone repository
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.pr-info.outputs.sha }}
          path: kickstart-tests

      - name: Check out kickstart-tests
        uses: actions/checkout@v4
        with:
          #TODO change
          #repository: rhinstaller/kickstart-tests
          repository: rvykydal/kickstart-tests
          path: kickstart-tests

      - name: Generate test cases
        working-directory: ./kickstart-tests
        run: scripts/generate-testcases.py -t ./testlib/test_cases/kstest-template.tc.yaml.j2 . -o ./testlib/test_cases

      - name: Clone Permian repository
        uses: actions/checkout@v4
        with:
          repository: rhinstaller/permian
          path: permian
          ref: main

      - name: Clone tplib repository
        uses: actions/checkout@v4
        with:
          repository: rhinstaller/tplib
          path: tplib

      - name: Ensure http proxy is running
        run: sudo containers/squid.sh start
        working-directory: ./kickstart-tests

      - name: Update container images used here
        run: |
          sudo podman pull quay.io/rhinstaller/kstest-runner:latest

      - name: Get changed tests
        working-directory: ./kickstart-tests
        id: get_changed_tests
        run: |
          git fetch origin
          # TODO: share with scripts/run-travis.sh
          CHANGED_TESTS=$(git diff --name-only origin/$TARGET_BRANCH ${{ needs.pr-info.outputs.sha }} -- *.ks.in $(find -maxdepth 1 -name '*.sh' -perm -u+x) | sed 's/\.ks\.in$//; s/\.sh$//' | sort -u | tr '\n' ' ')
          echo "changed_tests=${CHANGED_TESTS}" >> $GITHUB_OUTPUT

      - name: Get skipped tests for platform ${{ matrix.platform }}
        id: get_platform_skiptest
        working-directory: ./kickstart-tests
        run: |
          set -eux

          if [ ${{ matrix.platform }} == "daily-iso" ]; then
            echo "skip_tests=skip-on-fedora" >> $GITHUB_OUTPUT
          elif [ ${{ matrix.platform }} == "rawhide" ]; then
            echo "skip_tests=skip-on-fedora" >> $GITHUB_OUTPUT
          elif [ ${{ matrix.platform }} == "rhel8" ]; then
            echo "skip_tests=skip-on-rhel,skip-on-rhel-8" >> $GITHUB_OUTPUT
          elif [ ${{ matrix.platform }} == "rhel9" ]; then
            echo "skip_tests=skip-on-rhel,skip-on-rhel-9" >> $GITHUB_OUTPUT
          else
            echo "Platform is not supported by kickstart tests yet!"
            exit 1
          fi

      - name: Generate test selection
        id: generate_query
        working-directory: ./kickstart-tests
        run: |
          set -eux

          if [ -n "${{ needs.pr-info.outputs.launch_args }}" ]; then
            echo "Running tests required by the comment explicitly."
            LAUNCH_ARGS="${{ needs.pr-info.outputs.launch_args }}"
          elif [ -n "${{steps.get_changed_tests.outputs.changed_tests }}" ]; then
            echo "Running tests affected by changes."
            LAUNCH_ARGS="${{ steps.get_changed_tests.outputs.changed_tests }}"
          else
            echo "Running smoke tests."
            LAUNCH_ARGS="--testtype smoke"
          fi

          source ./containers/runner/skip-testtypes
          PERMIAN_QUERY=$(scripts/generate-permian-query.py \
             --skip-testtypes $SKIP_TESTTYPES_ANACONDA_PR \
             --skip-testtypes ${{ steps.get_platform_skiptest.outputs.skip_tests }} \
             ${LAUNCH_ARGS} )
          if [ $? == 0 ]; then
            echo "launch_args=$LAUNCH_ARGS" >> $GITHUB_OUTPUT
            echo "query=$PERMIAN_QUERY" >> $GITHUB_OUTPUT
          else
            echo "Parsing of the request arguments failed"
            exit 1
          fi

      # we post statuses manually as this does not run from a pull_request event
      # https://developer.github.com/v3/repos/statuses/#create-a-status
      - name: Create in-progress status
        uses: octokit/request-action@v2.x
        with:
          route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}'
          context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.launch_args }}'
          description: 'running tests with args ${{ steps.generate_query.outputs.launch_args }}'
          state: pending
          target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


      # Fetch boot.iso and configiure its local location
      - name: Fetch boot.iso if available for platform ${{ matrix.platform }}
        id: boot_iso_for_platform
        run: |
          set -eux
          BOOT_ISO_PATH="${{ github.workspace }}/${{ matrix.platform }}.boot.iso"
          BOOT_ISO_URL="file://$BOOT_ISO_PATH"
          if [ "${{ matrix.platform }}" == "daily-iso" ]; then
            ${{ github.workspace }}/kickstart-tests/containers/runner/fetch_daily_iso.sh $GITHUB_TOKEN $BOOT_ISO_PATH
            echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
          elif [ "${{ matrix.platform }}" == "rawhide" ]; then
            curl -L https://download.fedoraproject.org/pub/fedora/linux/development/rawhide/Server/x86_64/os/images/boot.iso --output $BOOT_ISO_PATH
            echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
          else
            echo "Boot.iso for ${{ matrix.platform }} can't be fetched."
            echo "boot_iso=" >> $GITHUB_OUTPUT
          fi

      # Configure location of installation repositories for the platform
      # Also default boot.iso is defined by the value of urls.installation_tree
      # of kstestParams event structure.
      - name: Set installation tree for the platform
        id: set_installation_urls
        run: |
          set -eux
          if [ "${{ matrix.platform }}" == "rhel8" ]; then
            echo "installation_tree=http://download.eng.bos.redhat.com/rhel-8/nightly/RHEL-8/latest-RHEL-8.10.0/compose/BaseOS/x86_64/os" >> $GITHUB_OUTPUT
            echo "modular_url=http://download.eng.bos.redhat.com/rhel-8/nightly/RHEL-8/latest-RHEL-8.10.0/compose/AppStream/x86_64/os" >> $GITHUB_OUTPUT
          elif [ "${{ matrix.platform }}" == "rhel9" ]; then
            echo "installation_tree=http://download.eng.bos.redhat.com/rhel-9/nightly/RHEL-9/latest-RHEL-9.4.0/compose/BaseOS/x86_64/os" >> $GITHUB_OUTPUT
            echo "modular_url=http://download.eng.bos.redhat.com/rhel-9/nightly/RHEL-9/latest-RHEL-9.4.0/compose/AppStream/x86_64/os" >> $GITHUB_OUTPUT
          else
            echo "Installation tree location for ${{ matrix.platform }} not configured"
            if [ -z "${{ steps.boot_iso_for_platform.outputs.boot_iso }}" ]; then
              echo "No boot.iso source is defined"
              exit 2
            fi
            echo "installation_tree=" >> $GITHUB_OUTPUT
            echo "modular_url=" >> $GITHUB_OUTPUT
          fi

      - name: Create Permian settings file
        working-directory: ./permian
        run: |
          cat <<EOF > settings.ini
          [kickstart_test]
          kstest_local_repo=${{ github.workspace }}/kickstart-tests
          [library]
          directPath=${{ github.workspace }}/kickstart-tests/testlib
          EOF

      - name: Run kickstart tests in container
        working-directory: ./permian
        run: |
          sudo --preserve-env=TEST_JOBS \
          PYTHONPATH=${PYTHONPATH:-}:${{ github.workspace }}/tplib \
          ./run_subset --debug-log permian.log \
            --settings settings.ini \
            --override workflows.dry_run=False \
            --testcase-query '${{ steps.generate_query.outputs.query }}' \
            run_event '{
              "type":"everything",
              "everything_testplan":{
                "configurations":[{"architecture":"x86_64"}],
                "point_person":"rvykydal@redhat.com"
              },
              ${{ steps.boot_iso_from_platform.outputs.boot_iso }}
              "kstestParams":{
                "platform":"${{ matrix.platform }}",
                "urls":{
                  "x86_64":{
                    "installation_tree":"${{ steps.set_installation_urls.outputs.installation_tree }}",
                    "modular_url":"${{ steps.set_installation_urls.outputs.modular_url }}"
                  }
                }
              }
            }'

          # Permian hides the exit code of launcher, so error out this step manually based on logs
          rc=$( awk '/Runner return code: /{ print $4 }' permian.log)
          if [ -n "$rc" ]; then
            exit $rc
          else
            exit 111
          fi

      - name: Collect anaconda logs
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: 'logs-${{ matrix.scenario }}'
          # skip the /anaconda subdirectories, too large
          path: |
            kickstart-tests/data/logs/kstest*.log
            kickstart-tests/data/logs/kstest.log.json
            kickstart-tests/data/logs/kstest-*/*.log
            kickstart-tests/data/logs/kstest-*/anaconda/lorax-packages.log

      - name: Collect Permian logs
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: 'logs-permian-${{ matrix.scenario }}'
          path: |
            permian/permian.log

  result:
    if: ${{ always() }}
    name: Set result status
    runs-on: ubuntu-latest
    env:
      STATUS_NAME: test-platforms
    needs: [platform, pr-info]
    steps:
      - name: Show overall status
        uses: octokit/request-action@v2.x
        with:
          route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}'
          context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.launch_args }}'
          description: 'finished'
          state: ${{ needs.platform.result }}
          target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}