test-os-versions.yml

# Run kickstart tests in a PR triggered by a "/test-os-versions" command from an organization member
#
# /test-os-versions - execute all tests affected by the PR or smoke tests if there is no such.
# Or specify tests by combination of test names and/or options:
# /test-os-versions --testtype TESTTYPE --skip-testtypes TYPE[,TYPE..] TEST1 TEST2
#
# For dry run use:
# /test-os-versions-dry
name: test-os-versions
on:
  issue_comment:
    types: [created]

permissions:
  contents: read
  statuses: write

jobs:
  pr-info:
    if: startsWith(github.event.comment.body, '/test-os-versions')
    runs-on: ubuntu-latest
    steps:
      - name: Query comment author repository permissions
        uses: octokit/request-action@v2.x
        id: user_permission
        with:
          route: GET /repos/${{ github.repository }}/collaborators/${{ github.event.sender.login }}/permission
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # restrict running of tests to users with admin or write permission for the repository
      # see https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#get-repository-permissions-for-a-user
      # store output if user is allowed in allowed_user job output so it has to be checked in downstream job
      - name: Check if user does have correct permissions
        if: contains('admin write', fromJson(steps.user_permission.outputs.data).permission)
        id: check_user_perm
        run: |
          echo "User '${{ github.event.sender.login }}' has permission '${{ fromJson(steps.user_permission.outputs.data).permission }}' allowed values: 'admin', 'write'"
          echo "allowed_user=true" >> $GITHUB_OUTPUT

      - name: Get information for pull request
        uses: octokit/request-action@v2.x
        id: pr_api
        with:
          route: GET /repos/${{ github.repository }}/pulls/${{ github.event.issue.number }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Parse comment arguments
        id: parse_comment_args
        # Do not use comment body directly in the shell command to avoid possible code injection.
        env:
          BODY: ${{ github.event.comment.body }}
        run: |
          # extract first line and cut out the "/test-os-versions" first word
          ARGS=$(echo "$BODY" | sed -n '1 s/^[^ ]* *//p' | sed 's/[[:space:]]*$//')
          echo "comment arguments are: $ARGS"
          echo "comment_args=${ARGS}" >> $GITHUB_OUTPUT
          # check for dry run mode
          DRY_RUN=False
          echo "$BODY" | grep -q "/test-os-versions-dry" && DRY_RUN=True
          echo "Dry run: ${DRY_RUN}"
          echo "dry_run=${DRY_RUN}" >> $GITHUB_OUTPUT

    outputs:
      allowed_user: ${{ steps.check_user_perm.outputs.allowed_user }}
      base_ref: ${{ fromJson(steps.pr_api.outputs.data).base.ref }}
      sha: ${{ fromJson(steps.pr_api.outputs.data).head.sha }}
      comment_args: ${{ steps.parse_comment_args.outputs.comment_args }}
      dry_run: ${{ steps.parse_comment_args.outputs.dry_run }}


  os-version:
    needs: pr-info
    if: needs.pr-info.outputs.allowed_user == 'true'
    name: Run on os version
    runs-on: [self-hosted, kstest-test]
    env:
      STATUS_NAME: test-os-versions
      TARGET_BRANCH: ${{ needs.pr-info.outputs.base_ref }}
      TEST_JOBS: 16
      GITHUB_TOKEN: /home/github/github-token
    strategy:
      matrix:
        os-version: [daily-iso, rawhide, rhel8, rhel9, rhel10]
      fail-fast: false

    steps:
      # self-hosted runners don't do this automatically; also useful to keep stuff around for debugging
      # need to run sudo as the launch script and the container create root/other user owned files
      - name: Clean up previous run
        run: |
          sudo podman ps -q --all --filter='ancestor=kstest-runner' | xargs -tr sudo podman rm -f
          sudo podman volume rm --all || true
          sudo rm -rf * .git

      - name: Check out kickstart-tests
        uses: actions/checkout@v4
        with:
          repository: rhinstaller/kickstart-tests
          ref: ${{ needs.pr-info.outputs.sha }}
          fetch-depth: 0
          path: kickstart-tests

      - name: Generate test cases
        working-directory: ./kickstart-tests
        run: scripts/generate-testcases.py -t ./testlib/test_cases/kstest-template.tc.yaml.j2 . -o ./testlib/test_cases

      - name: Clone Permian repository
        uses: actions/checkout@v4
        with:
          repository: rhinstaller/permian
          path: permian
          ref: main

      - name: Clone tplib repository
        uses: actions/checkout@v4
        with:
          repository: rhinstaller/tplib
          path: tplib

      - name: Ensure http proxy is running
        run: sudo containers/squid.sh start
        working-directory: ./kickstart-tests

      - name: Update container images used here
        run: |
          sudo podman pull quay.io/rhinstaller/kstest-runner:latest

      - name: Get changed tests
        working-directory: ./kickstart-tests
        id: get_changed_tests
        run: |
          set -eux

          BASE_COMMIT=$(git merge-base ${{ needs.pr-info.outputs.sha }} origin/${{ env.TARGET_BRANCH }})
          CHANGED_TESTS=$(git diff --name-only $BASE_COMMIT HEAD -- *.ks.in $(find -maxdepth 1 -name '*.sh' -perm -u+x) | sed 's/\.ks\.in$//; s/\.sh$//' | sort -u | tr '\n' ' ')
          echo "changed_tests=${CHANGED_TESTS}" >> $GITHUB_OUTPUT

      - name: Get skipped tests for os version ${{ matrix.os-version }}
        id: get_os_version_specs
        working-directory: ./kickstart-tests
        run: |
          set -eux

          source ./containers/runner/skip-testtypes
          if [ ${{ matrix.os-version }} == "daily-iso" ]; then
            echo "skip_tests=skip-on-fedora" >> $GITHUB_OUTPUT
            echo "disabled_testtypes=$SKIP_TESTTYPES_DAILY_ISO" >> $GITHUB_OUTPUT
            echo "platform=fedora_rawhide" >> $GITHUB_OUTPUT
          elif [ ${{ matrix.os-version }} == "rawhide" ]; then
            echo "skip_tests=skip-on-fedora" >> $GITHUB_OUTPUT
            echo "disabled_testtypes=$SKIP_TESTTYPES_RAWHIDE" >> $GITHUB_OUTPUT
            echo "platform=fedora_rawhide" >> $GITHUB_OUTPUT
          elif [ ${{ matrix.os-version }} == "rhel8" ]; then
            echo "skip_tests=skip-on-rhel,skip-on-rhel-8" >> $GITHUB_OUTPUT
            echo "disabled_testtypes=$SKIP_TESTTYPES_RHEL8" >> $GITHUB_OUTPUT
            echo "platform=rhel8" >> $GITHUB_OUTPUT
          elif [ ${{ matrix.os-version }} == "rhel9" ]; then
            echo "skip_tests=skip-on-rhel,skip-on-rhel-9" >> $GITHUB_OUTPUT
            echo "disabled_testtypes=$SKIP_TESTTYPES_RHEL9" >> $GITHUB_OUTPUT
            echo "platform=rhel9" >> $GITHUB_OUTPUT
          elif [ ${{ matrix.os-version }} == "rhel10" ]; then
            echo "skip_tests=skip-on-rhel,skip-on-rhel-10" >> $GITHUB_OUTPUT
            echo "disabled_testtypes=$SKIP_TESTTYPES_RHEL10" >> $GITHUB_OUTPUT
            echo "platform=rhel10" >> $GITHUB_OUTPUT
          else
            echo "Platform is not supported by kickstart tests yet!"
            exit 1
          fi

      - name: Generate test selection
        id: generate_query
        working-directory: ./kickstart-tests
        run: |
          set -eux

          CHANGED_TESTS="${{ steps.get_changed_tests.outputs.changed_tests }}"
          DISABLED_TESTTYPES="${{ steps.get_os_version_specs.outputs.disabled_testtypes }}"
          LAUNCH_ARGS="${{ needs.pr-info.outputs.launch_args }}"

          if [ -n "${LAUNCH_ARGS}" ]; then
            echo "description=Running tests required by the PR comment explicitly." >> $GITHUB_OUTPUT
          elif [ -n "${CHANGED_TESTS}" ]; then
            echo "description=Running tests affected by changes (excluding disabled)." >> $GITHUB_OUTPUT
            LAUNCH_ARGS="${CHANGED_TESTS}"
            if [ -n "${DISABLED_TESTTYPES}" ]; then
              LAUNCH_ARGS="--skip-testtypes ${DISABLED_TESTTYPES} ${LAUNCH_ARGS}"
            fi
          else
            echo "description=Running smoke tests (no affected tests found)." >> $GITHUB_OUTPUT
            LAUNCH_ARGS="--testtype smoke"
          fi

          source ./containers/runner/skip-testtypes
          PERMIAN_QUERY=$(scripts/generate-permian-query.py \
             --skip-testtypes $SKIP_TESTTYPES_ANACONDA_PR \
             --skip-testtypes ${{ steps.get_os_version_specs.outputs.skip_tests }} \
             ${LAUNCH_ARGS} )
          if [ $? == 0 ]; then
            echo "launch_args=$LAUNCH_ARGS" >> $GITHUB_OUTPUT
            echo "query=$PERMIAN_QUERY" >> $GITHUB_OUTPUT
          else
            echo "Parsing of the request arguments failed"
            exit 1
          fi

      # we post statuses manually as this does not run from a pull_request event
      # https://developer.github.com/v3/repos/statuses/#create-a-status
      - name: Create in-progress status
        uses: octokit/request-action@v2.x
        with:
          route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}'
          context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.launch_args }}'
          description: '${{ steps.generate_query.outputs.description }}'
          state: pending
          target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # Fetch boot.iso and configure its local location
      - name: Fetch boot.iso if available for os version ${{ matrix.os-version }}
        id: boot_iso_for_os_version
        run: |
          set -eux
          BOOT_ISO_PATH="${{ github.workspace }}/${{ matrix.os-version }}.boot.iso"
          BOOT_ISO_URL="file://$BOOT_ISO_PATH"
          if [ "${{ matrix.os-version }}" == "daily-iso" ]; then
            ${{ github.workspace }}/kickstart-tests/containers/runner/fetch_daily_iso.sh $GITHUB_TOKEN $BOOT_ISO_PATH
            echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
          elif [ "${{ matrix.os-version }}" == "rawhide" ]; then
            curl -L https://download.fedoraproject.org/pub/fedora/linux/development/rawhide/Everything/x86_64/os/images/boot.iso --output $BOOT_ISO_PATH
            echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
          else
            echo "Boot.iso for ${{ matrix.os-version }} can't be fetched."
            echo "boot_iso=" >> $GITHUB_OUTPUT
          fi

      # Configure location of installation repositories for the os version
      # Also default boot.iso is defined by the value of urls.installation_tree
      # of kstestParams event structure.
      - name: Set installation tree for the os version
        id: set_installation_urls
        working-directory: ./kickstart-tests
        run: |
          set -eux
          if [ "${{ matrix.os-version }}" == "rhel8" ] || \
             [ "${{ matrix.os-version }}" == "rhel9" ] || \
             [ "${{ matrix.os-version }}" == "rhel10" ]; then
            source ./scripts/defaults-${{ matrix.os-version }}.sh
            echo "installation_tree=${KSTEST_URL}" >> $GITHUB_OUTPUT
            echo "modular_url=${KSTEST_MODULAR_URL}" >> $GITHUB_OUTPUT
          else
            echo "Installation tree location for ${{ matrix.os-version }} not configured"
            if [ -z "${{ steps.boot_iso_for_os_version.outputs.boot_iso }}" ]; then
              echo "No boot.iso source is defined"
              exit 2
            fi
            echo "installation_tree=" >> $GITHUB_OUTPUT
            echo "modular_url=" >> $GITHUB_OUTPUT
          fi

      - name: Create Permian settings file
        working-directory: ./permian
        run: |
          cat <<EOF > settings.ini
          [kickstart_test]
          kstest_local_repo=${{ github.workspace }}/kickstart-tests
          [library]
          directPath=${{ github.workspace }}/kickstart-tests/testlib
          EOF

      - name: Run kickstart tests in container
        working-directory: ./permian
        run: |
          sudo --preserve-env=TEST_JOBS \
          PYTHONPATH=${PYTHONPATH:-}:${{ github.workspace }}/tplib \
          ./run_subset --debug-log permian.log \
            --settings settings.ini \
            --override workflows.dry_run=${{ needs.pr-info.outputs.dry_run }} \
            --testcase-query '${{ steps.generate_query.outputs.query }}' \
            run_event '{
              "type":"everything",
              "everything_testplan":{
                "configurations":[{"architecture":"x86_64"}],
                "point_person":"rvykydal@redhat.com"
              },
              ${{ steps.boot_iso_for_os_version.outputs.boot_iso }}
              "kstestParams":{
                "platform":"${{ steps.get_os_version_specs.outputs.platform }}",
                "urls":{
                  "x86_64":{
                    "installation_tree":"${{ steps.set_installation_urls.outputs.installation_tree }}",
                    "modular_url":"${{ steps.set_installation_urls.outputs.modular_url }}"
                  }
                }
              }
            }'

          # Permian hides the exit code of launcher, so error out this step manually based on logs
          rc=$( awk '/Runner return code: /{ print $4 }' permian.log)
          if [ -n "$rc" ]; then
            exit $rc
          else
            grep -q "All execution and reporting is done" permian.log || exit 111
          fi

      - name: Collect anaconda logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: 'logs-${{ matrix.os-version }}'
          # skip the /anaconda subdirectories, too large
          path: |
            kickstart-tests/data/logs/kstest*.log
            kickstart-tests/data/logs/kstest.log.json
            kickstart-tests/data/logs/kstest-*/*.log
            kickstart-tests/data/logs/kstest-*/anaconda/lorax-packages.log
            kickstart-tests/data/logs/kstest-*/original-ks.cfg

      - name: Collect Permian logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: 'logs-permian-${{ matrix.os-version }}'
          path: |
            permian/permian.log

  result:
    needs: [os-version, pr-info]
    if: ${{ always() && needs.pr-info.outputs.allowed_user == 'true' }}
    name: Set result status
    runs-on: ubuntu-latest
    env:
      STATUS_NAME: test-os-versions
    steps:
      - name: Show overall status
        uses: octokit/request-action@v2.x
        with:
          route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}'
          context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.launch_args }}'
          description: 'finished'
          state: ${{ needs.os-version.result }}
          target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}