test-os-variants.yml

# Run kickstart tests in a PR triggered by a "/test-os-variants" command from an organization member
#
# To run all tests changed in the PR (or just smoke tests if the PR changes no tests)
# use this in a PR comment:
#
#  /test-os-variants
#
#
# Alternatively it is also possible to specify tests to run explicitly:
#
#  /test-os-variants --testtype TESTTYPE --skip-testtypes TYPE[,TYPE..] TEST1 TEST2
#
# Examples:
#
# /test-os-variants keyboard lang
# ... run the tests specified by names (space separated)
#
# /test-os-variants --skip-testtypes gh123,gh765 keyboard lang
# ... run the tests specified by names while skipping tests with given types (comma separated)
#
# /test-os-variants --testtype network
# ... run all the tests of type network
#
# /test-os-variants --testtype network --skip-testtypes gh123,gh765
# ... run all the tests of type network while skipping tests with given types (comma separated)
#
# /test-os-variants --skip-testtypes gh123,gh765
# ... run all the tests while skipping tests with given types (comma separated)
#
#
# Lastly there is a dry-run mode, that just indicates which tests would be run
# (useful to check test grouping and overall infra health):
#
#  /test-os-variants-dry ...
#
#
name: test-os-variants
on:
  issue_comment:
    types: [created]

permissions:
  contents: read
  statuses: write

jobs:
  pr-info:
    if: startsWith(github.event.comment.body, '/test-os-variants')
    runs-on: ubuntu-latest
    steps:
      - name: Query comment author repository permissions
        uses: octokit/request-action@v2.x
        id: user_permission
        with:
          route: GET /repos/${{ github.repository }}/collaborators/${{ github.event.sender.login }}/permission
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # restrict running of tests to users with admin or write permission for the repository
      # see https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#get-repository-permissions-for-a-user
      # store output if user is allowed in allowed_user job output so it has to be checked in downstream job
      - name: Check if user does have correct permissions
        if: contains('admin write', fromJson(steps.user_permission.outputs.data).permission)
        id: check_user_perm
        run: |
          echo "User '${{ github.event.sender.login }}' has permission '${{ fromJson(steps.user_permission.outputs.data).permission }}' allowed values: 'admin', 'write'"
          echo "allowed_user=true" >> $GITHUB_OUTPUT

      - name: Get information for pull request
        uses: octokit/request-action@v2.x
        id: pr_api
        with:
          route: GET /repos/${{ github.repository }}/pulls/${{ github.event.issue.number }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Parse comment arguments
        id: parse_comment_args
        # Do not use comment body directly in the shell command to avoid possible code injection.
        env:
          BODY: ${{ github.event.comment.body }}
        run: |
          # extract first line and cut out the "/test-os-variants" first word
          ARGS=$(echo "$BODY" | sed -n '1 s/^[^ ]* *//p' | sed 's/[[:space:]]*$//')
          echo "comment arguments are: $ARGS"
          echo "comment_args=${ARGS}" >> $GITHUB_OUTPUT
          # check for dry run mode
          DRY_RUN=False
          echo "$BODY" | grep -q "/test-os-variants-dry" && DRY_RUN=True
          echo "Dry run: ${DRY_RUN}"
          echo "dry_run=${DRY_RUN}" >> $GITHUB_OUTPUT

    outputs:
      allowed_user: ${{ steps.check_user_perm.outputs.allowed_user }}
      base_ref: ${{ fromJson(steps.pr_api.outputs.data).base.ref }}
      sha: ${{ fromJson(steps.pr_api.outputs.data).head.sha }}
      comment_args: ${{ steps.parse_comment_args.outputs.comment_args }}
      dry_run: ${{ steps.parse_comment_args.outputs.dry_run }}


  os-variant:
    needs: pr-info
    if: needs.pr-info.outputs.allowed_user == 'true'
    name: Run on os variant
    runs-on: [self-hosted, kstest]
    env:
      STATUS_NAME: test-os-variants
      TARGET_BRANCH: ${{ needs.pr-info.outputs.base_ref }}
      TEST_JOBS: 16
      GITHUB_TOKEN: /home/github/github-token
    strategy:
      matrix:
        os-variant: [daily-iso, rawhide, rhel9, rhel10]
      fail-fast: false

    steps:
      # self-hosted runners don't do this automatically; also useful to keep stuff around for debugging
      # need to run sudo as the launch script and the container create root/other user owned files
      - name: Clean up previous run
        run: |
          sudo podman ps -q --all --filter='ancestor=kstest-runner' | xargs -tr sudo podman rm -f
          sudo podman volume rm --all || true
          sudo rm -rf * .git

      - name: Check out kickstart-tests
        uses: actions/checkout@v4
        with:
          repository: rhinstaller/kickstart-tests
          ref: ${{ needs.pr-info.outputs.sha }}
          fetch-depth: 0
          path: kickstart-tests

      - name: Generate test cases
        working-directory: ./kickstart-tests
        run: scripts/generate-testcases.py -t ./testlib/test_cases/kstest-template.tc.yaml.j2 . -o ./testlib/test_cases

      - name: Clone Permian repository
        uses: actions/checkout@v4
        with:
          repository: rhinstaller/permian
          path: permian
          ref: main

      - name: Clone tplib repository
        uses: actions/checkout@v4
        with:
          repository: rhinstaller/tplib
          path: tplib

      - name: Ensure http proxy is running
        run: sudo containers/squid.sh start
        working-directory: ./kickstart-tests

      - name: Update container images used here
        run: |
          sudo podman pull quay.io/rhinstaller/kstest-runner:latest

      - name: Get changed tests
        working-directory: ./kickstart-tests
        id: get_changed_tests
        run: |
          set -eux

          BASE_COMMIT=$(git merge-base ${{ needs.pr-info.outputs.sha }} origin/${{ env.TARGET_BRANCH }})
          CHANGED_TESTS=$(git diff --name-only $BASE_COMMIT HEAD -- *.ks.in $(find -maxdepth 1 -name '*.sh' -perm -u+x) | sed 's/\.ks\.in$//; s/\.sh$//' | sort -u | tr '\n' ' ')
          echo "changed_tests=${CHANGED_TESTS}" >> $GITHUB_OUTPUT

      - name: Generate test selection for os variant ${{ matrix.os-variant }}
        id: generate_query
        working-directory: ./kickstart-tests
        run: |
          CHANGED_TESTS="${{ steps.get_changed_tests.outputs.changed_tests }}"
          COMMENT_ARGS="${{ needs.pr-info.outputs.comment_args }}"

          if [ -n "${COMMENT_ARGS}" ]; then
            echo "description=Running tests required by the PR comment explicitly." >> $GITHUB_OUTPUT
            COMMENT_ARGS="--force ${COMMENT_ARGS}"
          elif [ -n "${CHANGED_TESTS}" ]; then
            echo "description=Running tests affected by changes (excluding disabled)." >> $GITHUB_OUTPUT
            COMMENT_ARGS="${CHANGED_TESTS}"
          else
            echo "description=Running smoke tests (no affected tests found)." >> $GITHUB_OUTPUT
            COMMENT_ARGS="--testtype smoke"
          fi

          LAUNCH_ARGS=$(scripts/generate-launch-args.py ${COMMENT_ARGS} \
             --os-variant ${{ matrix.os-variant }} ) || RC=$?
          if [ -z ${RC} ] || [ ${RC} == 0 ]; then
            echo "Generated launch arguments: $LAUNCH_ARGS"
          else
            echo "Generating of the arguments failed. See the workflow file for usage."
            exit 1
          fi
          PERMIAN_QUERY=$(scripts/generate-permian-query.py $LAUNCH_ARGS)
          echo "Generated permian query: $PERMIAN_QUERY"
          echo "query=$PERMIAN_QUERY" >> $GITHUB_OUTPUT
          PLATFORM=$(scripts/generate-permian-query.py --print-platform $LAUNCH_ARGS)
          echo "Generated platform: $PLATFORM"
          echo "platform=$PLATFORM" >> $GITHUB_OUTPUT

      # we post statuses manually as this does not run from a pull_request event
      # https://developer.github.com/v3/repos/statuses/#create-a-status
      - name: Create in-progress status
        uses: octokit/request-action@v2.x
        with:
          route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}'
          context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.comment_args }}'
          description: '${{ steps.generate_query.outputs.description }}'
          state: pending
          target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # Fetch boot.iso and configure its local location
      - name: Fetch boot.iso if available for os variant ${{ matrix.os-variant }}
        id: boot_iso_for_os_variant
        run: |
          set -eux
          BOOT_ISO_PATH="${{ github.workspace }}/${{ matrix.os-variant }}.boot.iso"
          BOOT_ISO_URL="file://$BOOT_ISO_PATH"
          if [ "${{ matrix.os-variant }}" == "daily-iso" ]; then
            ${{ github.workspace }}/kickstart-tests/containers/runner/fetch_daily_iso.sh $GITHUB_TOKEN $BOOT_ISO_PATH
            echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
          elif [ "${{ matrix.os-variant }}" == "rawhide" ]; then
            curl -L https://download.fedoraproject.org/pub/fedora/linux/development/rawhide/Everything/x86_64/os/images/boot.iso --output $BOOT_ISO_PATH
            echo "boot_iso=\"bootIso\":{\"x86_64\":\"${BOOT_ISO_URL}\"}," >> $GITHUB_OUTPUT
          else
            echo "Boot.iso for ${{ matrix.os-variant }} can't be fetched."
            echo "boot_iso=" >> $GITHUB_OUTPUT
          fi

      # Configure location of installation repositories for the os variant
      # Also default boot.iso is defined by the value of urls.installation_tree
      # of kstestParams event structure.
      - name: Set installation tree for the os variant
        id: set_installation_urls
        working-directory: ./kickstart-tests
        run: |
          set -eux
          if [ "${{ matrix.os-variant }}" == "rhel8" ] || \
             [ "${{ matrix.os-variant }}" == "rhel9" ] || \
             [ "${{ matrix.os-variant }}" == "rhel10" ]; then
            source ./scripts/defaults-${{ matrix.os-variant }}.sh
            echo "installation_tree=${KSTEST_URL}" >> $GITHUB_OUTPUT
            echo "modular_url=${KSTEST_MODULAR_URL}" >> $GITHUB_OUTPUT
          else
            echo "Installation tree location for ${{ matrix.os-variant }} not configured"
            if [ -z "${{ steps.boot_iso_for_os_variant.outputs.boot_iso }}" ]; then
              echo "No boot.iso source is defined"
              exit 2
            fi
            echo "installation_tree=" >> $GITHUB_OUTPUT
            echo "modular_url=" >> $GITHUB_OUTPUT
          fi

      - name: Create Permian settings file
        working-directory: ./permian
        run: |
          cat <<EOF > settings.ini
          [kickstart_test]
          kstest_local_repo=${{ github.workspace }}/kickstart-tests
          [library]
          directPath=${{ github.workspace }}/kickstart-tests/testlib
          EOF

      - name: Run kickstart tests in container
        working-directory: ./permian
        run: |
          sudo --preserve-env=TEST_JOBS \
          PYTHONPATH=${PYTHONPATH:-}:${{ github.workspace }}/tplib \
          ./run_subset --debug-log permian.log \
            --settings settings.ini \
            --override workflows.dry_run=${{ needs.pr-info.outputs.dry_run }} \
            --testcase-query '${{ steps.generate_query.outputs.query }}' \
            run_event '{
              "type":"everything",
              "everything_testplan":{
                "configurations":[{"architecture":"x86_64"}],
                "point_person":"rvykydal@redhat.com"
              },
              ${{ steps.boot_iso_for_os_variant.outputs.boot_iso }}
              "kstestParams":{
                "platform":"${{ steps.generate_query.outputs.platform }}",
                "urls":{
                  "x86_64":{
                    "installation_tree":"${{ steps.set_installation_urls.outputs.installation_tree }}",
                    "modular_url":"${{ steps.set_installation_urls.outputs.modular_url }}"
                  }
                }
              }
            }'

          # Permian hides the exit code of launcher, so error out this step manually based on logs
          rc=$( awk '/Runner return code: /{ print $4 }' permian.log)
          if [ -n "$rc" ]; then
            exit $rc
          else
            grep -q "All execution and reporting is done" permian.log || exit 111
          fi

      - name: Collect anaconda logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: 'logs-${{ matrix.os-variant }}'
          # skip the /anaconda subdirectories, too large
          path: |
            kickstart-tests/data/logs/kstest*.log
            kickstart-tests/data/logs/kstest.log.json
            kickstart-tests/data/logs/kstest-*/*.log
            kickstart-tests/data/logs/kstest-*/anaconda/lorax-packages.log
            kickstart-tests/data/logs/kstest-*/original-ks.cfg

      - name: Collect Permian logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: 'logs-permian-${{ matrix.os-variant }}'
          path: |
            permian/permian.log

  result:
    needs: [os-variant, pr-info]
    if: ${{ always() && needs.pr-info.outputs.allowed_user == 'true' }}
    name: Set result status
    runs-on: ubuntu-latest
    env:
      STATUS_NAME: test-os-variants
    steps:
      - name: Show overall status
        uses: octokit/request-action@v2.x
        with:
          route: 'POST /repos/${{ github.repository }}/statuses/${{ needs.pr-info.outputs.sha }}'
          context: '${{ env.STATUS_NAME }} ${{ needs.pr-info.outputs.comment_args }}'
          description: 'finished'
          state: ${{ needs.os-variant.result }}
          target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}