From 090564687f975af6434d1219084adb707c66611e Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 9 Oct 2024 16:04:23 -0400 Subject: [PATCH 01/16] DAOS-16262 build: Add weekly bullseye build and test GHA Automate weekly build with Bullseye and VM Functionality test. Skip-unit-tests: true Skip-fault-injection-test: true Skip-test: true Required-githooks: true Signed-off-by: Phil Henderson --- .github/workflows/code-coverage-test.yml | 418 +++++++++++++++++++++++ 1 file changed, 418 insertions(+) create mode 100644 .github/workflows/code-coverage-test.yml diff --git a/.github/workflows/code-coverage-test.yml b/.github/workflows/code-coverage-test.yml new file mode 100644 index 00000000000..804ef1d6b1b --- /dev/null +++ b/.github/workflows/code-coverage-test.yml @@ -0,0 +1,418 @@ +name: Code Coverage Test + +env: + # TODO: we really need to define a list of supported versions (ideally it's no more than 2) + # build is done on the lowest version and test on the highest with a "sanity test" + # stage done on all versions in the list except the highest + EL8_BUILD_VERSION: 8.6 + EL8_VERSION: 8.8 + EL9_BUILD_VERSION: 9 + EL9_VERSION: 9 + LEAP15_VERSION: 15.5 + + JENKINS_URL: https://build.hpdd.intel.com/ + +on: + pull_request: # Remove after testing + schedule: + - cron: "0 0 * * 6" # Every Saturday + +concurrency: + group: code-coverage-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash --noprofile --norc -ueo pipefail {0} + +permissions: {} + +jobs: + + Build-RPM: + name: Bullseye Build RPM + permissions: + statuses: write + runs-on: [self-hosted, docker] + if: ((!cancelled()) || success() || failure()) + strategy: + matrix: + distro: ["el8"] + fail-fast: false + env: + ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ + DAOS_EMAIL: phil.henderson@intel.com + DAOS_FULLNAME: daos-stack + DISTRO: ${{ matrix.distro }} + DISTRO_REPOS: disabled + # DOCKER_BUILDKIT: 0 + ARTIFACTS_URL: file:///scratch/job_repos/ + MOCK_OPTIONS: --uniqueext=${{ github.run_id }} + PR_NUM: ${{ github.event.pull_request.number }} + # TODO -- this should be on stable, backed up storage, not /scratch + # yamllint disable-line rule:line-length + REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ github.event.pull_request.number }}/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + RUN_ID: ${{ github.run_id }} + TARGET: ${{ matrix.distro }} + # keep VS Code's GHA linting happy + STAGE_NAME: + DISTRO_NAME: + DISTRO_VERSION: + CP_LEAP15_VERSION: + COMMIT_STATUS_DISTRO_VERSION: + FVERSION: + steps: + # - name: Import commit pragmas + # uses: ./.github/actions/import-commit-pragmas + - name: Set variables + run: | + FVERSION="38" + case ${{ matrix.distro }} in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}" + COMMIT_STATUS_DISTRO_VERSION="8" + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}" + ;; + 'leap15') + CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || + env.LEAP15_VERSION }}-x86_64" + DISTRO_NAME="Leap" + DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" + ;; + esac + echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV + echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV + echo "FVERSION=$FVERSION" >> $GITHUB_ENV + echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV + echo "COVFN_DISABLED=false" >> $GITHUB_ENV + echo "BULLSEYE=yes" >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Build RPM Docker image + id: build-rpm-docker-image + continue-on-error: true + run: docker build --file utils/rpms/packaging/Dockerfile.mockbuild + --build-arg CACHEBUST=$(date +%s%3N) + --build-arg CB0=$(date +%V) + --build-arg REPO_FILE_URL=$REPO_FILE_URL + --build-arg UID=$(id -u) + --build-arg FVERSION=${{ env.FVERSION }} + --build-arg JENKINS_URL=${{ env.JENKINS_URL }} + --build-arg BULLSEYE_KEY=${{ secrets.BULLSEYE_LICENSE_KEY }} + --tag mock-build + utils/rpms + - name: Build RPM + id: build-rpm + continue-on-error: true + # yamllint disable rule:line-length + run: rm -rf mock_result; + mkdir -p mock_result; + docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }} + --user build + -v "$PWD":"$PWD" -w "$PWD" + -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result + --privileged=true + -e DAOS_FULLNAME="$DAOS_FULLNAME" + -e DAOS_EMAIL="$DAOS_EMAIL" + -e DISTRO_VERSION="$DISTRO_VERSION" + -e STAGE_NAME="$STAGE_NAME" + -e CHROOT_NAME="$CHROOT_NAME" + -e ARTIFACTORY_URL="$ARTIFACTORY_URL" + -e REPO_FILE_URL="$REPO_FILE_URL" + -e JENKINS_URL=${{ env.JENKINS_URL }} + -e TARGET="$TARGET" + -e COVFN_DISABLED="$COVFN_DISABLED" + mock-build ci/rpm/build.sh + # yamllint enable rule:line-length + - name: Build RPM failure log + id: build-rpm-fail-log + continue-on-error: true + if: steps.build-rpm.outcome != 'success' + run: cat mock_result/root.log; + cat mock_result/build.log + - name: Save RPM build logs + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} logs + path: | + mock_result/root.log + mock_result/build.log + - name: Create lastBuild and lastSuccessfulBuild symlinks + if: steps.build-rpm.outcome == 'success' + run: . ci/gha_functions.sh; + mkdir -p ${REPO_PATH}; + rm -f ${REPO_PATH}last{,Successful}Build; + ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild; + ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild + - name: Create repo + id: create-repo + if: steps.build-rpm.outcome == 'success' + continue-on-error: true + run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh + - name: Test repo + id: test-repo + if: steps.create-repo.outcome == 'success' + continue-on-error: true + run: . ci/gha_functions.sh; + dnf --disablerepo=\* --repofrompath + testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET + repoquery -a + - name: Remove lastSuccessfulBuild link and exit failure + if: steps.test-repo.outcome != 'success' + run: rm -f ${REPO_PATH}lastSuccessfulBuild; + exit 1 + - name: Publish RPMs + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} RPM repository + path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} + - name: Update commit status + uses: ouzi-dev/commit-status-updater@v2 + with: + # yamllint disable-line rule:line-length + name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' + status: "${{ job.status }}" + + # Unit-Test: + # name: Bullseye Unit Test + # runs-on: [self-hosted, docker] + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + # with: + # submodules: true + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Build deps in Docker + # run: docker build . --file utils/docker/Dockerfile.el.8 + # --build-arg DAOS_BUILD=no + # --build-arg DEPS_JOBS=50 + # - name: Build daos in Docker + # run: docker build . --file utils/docker/Dockerfile.el.8 + # --build-arg DEPS_JOBS=50 + # --build-arg DAOS_KEEP_SRC=yes + # --build-arg DAOS_TARGET_TYPE=debug + # --build-arg DAOS_JAVA_BUILD=yes + # --build-arg JENKINS_URL=${{ env.JENKINS_URL }} + # --build-arg BULLSEYE=${{ secrets.BULLSEYE_LICENSE_KEY }} + # --tag build-image-${{github.run_id}}-${{github.run_attempt}} + # - name: Run Unit Testing + # run: docker exec -t build-image-${{github.run_id}}-${{github.run_attempt}} + # sh -c "ls -al"; + # docker run --name unit-test-${{github.run_id}}-${{github.run_attempt}} + # -e STAGE_NAME="Unit Test Bullseye" + # -e BULLSEYE=${{ secrets.BULLSEYE_LICENSE_KEY }} + # build-image-${{github.run_id}}-${{github.run_attempt}} + # ci/unit/test_main_node.sh + # - name: Fetch test results + # run: docker cp unit-test-${{github.run_id}}-${{github.run_attempt}}:/covc_test_logs + # ./artifacts; + # docker cp unit-test-${{github.run_id}}-${{github.run_attempt}}:/covc_vm_test + # ./artifacts + # - name: Upload test results + # uses: actions/upload-artifact@v4 + # with: + # name: Bullseye Unit Testing test-results + # path: ./artifacts/ + + Functional-Test: + name: Bullseye Functional Test + runs-on: [self-hosted, wolf] + permissions: + statuses: write + # https://github.com/EnricoMi/publish-unit-test-result-action#permissions + checks: write + pull-requests: write + timeout-minutes: 7200 + needs: [Build-RPM] + strategy: + matrix: + distro: ["el8"] + fail-fast: false + # https://github.com/actions/runner/issues/491#issuecomment-926924523 + if: | + (!cancelled()) && + (needs.Build-RPM.result == 'success' || + needs.Build-RPM.result == 'skipped') + env: + CONFIG_POWER_ONLY: false + PRAGMA_SUFFIX: -vm + OPERATIONS_EMAIL: phil.henderson@intel.com + TEST_RPMS: true + COMMIT_MESSAGE: + JENKINS_URL: https://build.hpdd.intel.com/ + REPOSITORY_URL: https://repo.dc.hpdd.intel.com/ + REMOVE_EXISTING_RPMS: false + # TODO -- this should be on stable, backedup storage + ARTIFACTS_URL: file:///scratch/job_repos/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + # keep VS Code's GHA linting happy + NODESTRING: + CP_PR_REPOS: + CP_FEATURES: + CP_TEST_TAG: + CP_EL8_VM9_LABEL: + CP_EL9_VM9_LABEL: + CP_LEAP15_VM9_LABEL: + CP_PRIORITY: + CP_EL8_VERSION: + CP_EL9_VERSION: + CP_LEAP15_VERSION: + DISTRO: + CLUSTER_REQUEST_reqid: + STAGE_NAME: + QUEUE_URL: + LABEL: + DISTRO_NAME: + DISTRO_VERSION: + COMMIT_STATUS_DISTRO_VERSION: + FTEST_ARG: + steps: + # - name: Import commit pragmas + # uses: ./.github/actions/import-commit-pragmas + - name: Set variables + run: | + set -eux + env + STAGE_TAGS="vm" + FTEST_ARG="--nvme=auto" + INST_RPMS="daos-client daos-tests daos-server daos-serialize daos-tests-internal" + case "${{ matrix.distro }}" in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION="${{ env.CP_EL8_VERSION && + env.CP_EL8_VERSION || env.EL8_VERSION }}" + DISTRO_VERSION_MAJOR="8" + OPENMPI="openmpi" + LABEL="${{ env.CP_EL8_VM9_LABEL && + env.CP_EL8_VM9_LABEL || 'ci_vm9' }}" + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION="${{ env.CP_EL9_VERSION && + env.CP_EL9_VERSION || env.EL9_VERSION }}" + DISTRO_VERSION_MAJOR="9" + PROV_DISTRO_VERSION_MAJOR="8" + OPENMPI="openmpi" + LABEL="${{ env.CP_EL9_VM9_LABEL && + env.CP_EL9_VM9_LABEL || 'ci_vm9' }}" + ;; + 'leap15') + CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || + env.LEAP15_VERSION }}-x86_64" + DISTRO_NAME="Leap" + DISTRO_NAME_UPPER="LEAP" + DISTRO_NAME_LOWER="leap" + DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" + DISTRO_VERSION_MAJOR="15" + OPENMPI="openmpi3" + LABEL="${{ env.CP_LEAP15_VM9_LABEL && + env.CP_LEAP15_VM9_LABEL || 'ci_vm9' }}" + ;; + esac + echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV + echo "STAGE_NAME=Functional on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV + echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV + echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV + echo "DISTRO=${DISTRO_NAME_UPPER}_$DISTRO_VERSION_MAJOR" >> $GITHUB_ENV + echo -n "PROVISION_DISTRO=${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}" >> $GITHUB_ENV + echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ + $GITHUB_ENV + echo "LABEL=$LABEL" >> $GITHUB_ENV + echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 500 + ref: ${{ github.event.pull_request.head.sha }} + - name: Request and Provision a Cluster + timeout-minutes: 7200 + uses: ./.github/actions/provision-cluster + with: + condition: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && \ + env.CP_SKIP_FUNC_TEST != 'true' + - name: Run Test + timeout-minutes: 7200 + if: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && env.CP_SKIP_FUNC_TEST != 'true' + id: run-test + run: | + . ci/gha_functions.sh + NODE_COUNT="$NODE_COUNT" \ + TEST_TAG="pr,vm" \ + FTEST_ARG="${{ env.FTEST_ARG }}" ci/functional/test_main.sh + - name: Cancel cluster request (if cancelled after requesting) + if: cancelled() + run: | + set -eux + . ci/gha_functions.sh + if ! JENKINS_URL="${{ env.JENKINS_URL }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ + cancel_provision; then + # probably already provisioned and needs unprovisioning + if ! cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}"; then + exit 1 + fi + fi + - name: Job cleanup + if: (!cancelled() && (success() || failure())) + run: | + set -eux + . ci/gha_functions.sh + NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh || true + cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}" + - name: Publish test results + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + check_name: ${{ env.STAGE_NAME }} Test Results (old) + github_token: ${{ secrets.GITHUB_TOKEN }} + junit_files: ${{ env.STAGE_NAME }}/**/results.xml + - name: Publish artifacts + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} artifacts + path: ${{ env.STAGE_NAME }}/** + - name: Upload test results + if: (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} test-results + path: ${{ env.STAGE_NAME }}/**/results.xml + - name: Update commit status + uses: ouzi-dev/commit-status-updater@v2 + with: + # yamllint disable-line rule:line-length + name: 'test/Functional on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' + status: "${{ job.status }}" From b973d2c1c525021e1b971248699bd7f813968759 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 11 Oct 2024 16:45:05 -0400 Subject: [PATCH 02/16] Avoid code re-use with compositre actions. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/build-rpm/action.yml | 178 +++++++++ .github/actions/functional-test/action.yml | 195 ++++++++++ .github/workflows/code-coverage-test.yml | 412 +-------------------- 3 files changed, 387 insertions(+), 398 deletions(-) create mode 100644 .github/actions/build-rpm/action.yml create mode 100644 .github/actions/functional-test/action.yml diff --git a/.github/actions/build-rpm/action.yml b/.github/actions/build-rpm/action.yml new file mode 100644 index 00000000000..efdab10d070 --- /dev/null +++ b/.github/actions/build-rpm/action.yml @@ -0,0 +1,178 @@ +name: 'Build RPM' +description: 'Build DAOS RPMs' + +env: + # TODO: we really need to define a list of supported versions (ideally it's no more than 2) + # build is done on the lowest version and test on the highest with a "sanity test" + # stage done on all versions in the list except the highest + EL8_BUILD_VERSION: 8.6 + EL8_VERSION: 8.8 + EL9_BUILD_VERSION: 9 + EL9_VERSION: 9 + LEAP15_VERSION: 15.5 + +on: + workflow_call: + inputs: + distro: + type: choice + options: + - el8 + - el9 + - leap15 + covfn_disabled: + type: bool + default: true + +jobs: + Build-RPM: + name: Bullseye Build RPM + permissions: + statuses: write + runs-on: [self-hosted, docker] + if: ((!cancelled()) || success() || failure()) + env: + ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ + JENKINS_URL: https://build.hpdd.intel.com/ + DAOS_EMAIL: phil.henderson@intel.com + DAOS_FULLNAME: daos-stack + DISTRO: ${{ inputs.distro }} + DISTRO_REPOS: disabled + # DOCKER_BUILDKIT: 0 + ARTIFACTS_URL: file:///scratch/job_repos/ + MOCK_OPTIONS: --uniqueext=${{ github.run_id }} + PR_NUM: ${{ github.event.pull_request.number }} + # TODO -- this should be on stable, backed up storage, not /scratch + # yamllint disable-line rule:line-length + REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ github.event.pull_request.number }}/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + RUN_ID: ${{ github.run_id }} + TARGET: ${{ inputs.distro }} + # keep VS Code's GHA linting happy + STAGE_NAME: + DISTRO_NAME: + DISTRO_VERSION: + CP_LEAP15_VERSION: + COMMIT_STATUS_DISTRO_VERSION: + FVERSION: + steps: + - name: Set variables + run: | + FVERSION="38" + case ${{ inputs.distro }} in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}" + COMMIT_STATUS_DISTRO_VERSION="8" + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}" + ;; + 'leap15') + CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || + env.LEAP15_VERSION }}-x86_64" + DISTRO_NAME="Leap" + DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" + ;; + esac + echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV + echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV + echo "FVERSION=$FVERSION" >> $GITHUB_ENV + echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV + echo "COVFN_DISABLED=${{ inputs.covfn_disabled }}" >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Build RPM Docker image + id: build-rpm-docker-image + continue-on-error: true + run: docker build --file utils/rpms/packaging/Dockerfile.mockbuild + --build-arg CACHEBUST=$(date +%s%3N) + --build-arg CB0=$(date +%V) + --build-arg REPO_FILE_URL=$REPO_FILE_URL + --build-arg UID=$(id -u) + --build-arg FVERSION=${{ env.FVERSION }} + --build-arg JENKINS_URL=${{ env.JENKINS_URL }} + --tag mock-build + utils/rpms + - name: Build RPM + id: build-rpm + continue-on-error: true + # yamllint disable rule:line-length + run: rm -rf mock_result; + mkdir -p mock_result; + docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.distro }} + --user build + -v "$PWD":"$PWD" -w "$PWD" + -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result + --privileged=true + -e DAOS_FULLNAME="$DAOS_FULLNAME" + -e DAOS_EMAIL="$DAOS_EMAIL" + -e DISTRO_VERSION="$DISTRO_VERSION" + -e STAGE_NAME="$STAGE_NAME" + -e CHROOT_NAME="$CHROOT_NAME" + -e ARTIFACTORY_URL="$ARTIFACTORY_URL" + -e REPO_FILE_URL="$REPO_FILE_URL" + -e JENKINS_URL=${{ env.JENKINS_URL }} + -e TARGET="$TARGET" + -e COVFN_DISABLED="$COVFN_DISABLED" + mock-build ci/rpm/build.sh + # yamllint enable rule:line-length + - name: Build RPM failure log + id: build-rpm-fail-log + continue-on-error: true + if: steps.build-rpm.outcome != 'success' + run: cat mock_result/root.log; + cat mock_result/build.log + - name: Save RPM build logs + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} logs + path: | + mock_result/root.log + mock_result/build.log + - name: Create lastBuild and lastSuccessfulBuild symlinks + if: steps.build-rpm.outcome == 'success' + run: . ci/gha_functions.sh; + mkdir -p ${REPO_PATH}; + rm -f ${REPO_PATH}last{,Successful}Build; + ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild; + ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild + - name: Create repo + id: create-repo + if: steps.build-rpm.outcome == 'success' + continue-on-error: true + run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh + - name: Test repo + id: test-repo + if: steps.create-repo.outcome == 'success' + continue-on-error: true + run: . ci/gha_functions.sh; + dnf --disablerepo=\* --repofrompath + testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET + repoquery -a + - name: Remove lastSuccessfulBuild link and exit failure + if: steps.test-repo.outcome != 'success' + run: rm -f ${REPO_PATH}lastSuccessfulBuild; + exit 1 + - name: Publish RPMs + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} RPM repository + path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} + - name: Update commit status + uses: ouzi-dev/commit-status-updater@v2 + with: + # yamllint disable-line rule:line-length + name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' + status: "${{ job.status }}" diff --git a/.github/actions/functional-test/action.yml b/.github/actions/functional-test/action.yml new file mode 100644 index 00000000000..5ac043110e5 --- /dev/null +++ b/.github/actions/functional-test/action.yml @@ -0,0 +1,195 @@ +name: 'Functional Test' +description: 'Run VM functional tests against DAOS RPMs' + +on: + workflow_call: + inputs: + distro: + type: choice + options: + - el8 + - el9 + - leap15 + tags: + type: string + required: true + ftest_args: + type: string + default: --nvme=auto + +jobs: + runs-on: [self-hosted, wolf] + permissions: + statuses: write + # https://github.com/EnricoMi/publish-unit-test-result-action#permissions + checks: write + pull-requests: write + timeout-minutes: 7200 + env: + CONFIG_POWER_ONLY: false + PRAGMA_SUFFIX: -vm + OPERATIONS_EMAIL: phil.henderson@intel.com + TEST_RPMS: true + COMMIT_MESSAGE: + JENKINS_URL: https://build.hpdd.intel.com/ + REPOSITORY_URL: https://repo.dc.hpdd.intel.com/ + REMOVE_EXISTING_RPMS: false + # TODO -- this should be on stable, backedup storage + ARTIFACTS_URL: file:///scratch/job_repos/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + # keep VS Code's GHA linting happy + NODESTRING: + CP_PR_REPOS: + CP_FEATURES: + CP_TEST_TAG: + CP_EL8_VM9_LABEL: + CP_EL9_VM9_LABEL: + CP_LEAP15_VM9_LABEL: + CP_PRIORITY: + CP_EL8_VERSION: + CP_EL9_VERSION: + CP_LEAP15_VERSION: + DISTRO: + CLUSTER_REQUEST_reqid: + STAGE_NAME: + QUEUE_URL: + LABEL: + DISTRO_NAME: + DISTRO_VERSION: + COMMIT_STATUS_DISTRO_VERSION: + FTEST_ARG: + steps: + # - name: Import commit pragmas + # uses: ./.github/actions/import-commit-pragmas + - name: Set variables + run: | + set -eux + env + STAGE_TAGS="vm" + FTEST_ARG={{ $inputs.ftest_args }} + INST_RPMS="daos-client daos-tests daos-server daos-serialize daos-tests-internal" + case "${{ inputs.distro }}" in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION="${{ env.CP_EL8_VERSION && + env.CP_EL8_VERSION || env.EL8_VERSION }}" + DISTRO_VERSION_MAJOR="8" + OPENMPI="openmpi" + LABEL="${{ env.CP_EL8_VM9_LABEL && + env.CP_EL8_VM9_LABEL || 'ci_vm9' }}" + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION="${{ env.CP_EL9_VERSION && + env.CP_EL9_VERSION || env.EL9_VERSION }}" + DISTRO_VERSION_MAJOR="9" + PROV_DISTRO_VERSION_MAJOR="8" + OPENMPI="openmpi" + LABEL="${{ env.CP_EL9_VM9_LABEL && + env.CP_EL9_VM9_LABEL || 'ci_vm9' }}" + ;; + 'leap15') + CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || + env.LEAP15_VERSION }}-x86_64" + DISTRO_NAME="Leap" + DISTRO_NAME_UPPER="LEAP" + DISTRO_NAME_LOWER="leap" + DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" + DISTRO_VERSION_MAJOR="15" + OPENMPI="openmpi3" + LABEL="${{ env.CP_LEAP15_VM9_LABEL && + env.CP_LEAP15_VM9_LABEL || 'ci_vm9' }}" + ;; + esac + echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV + echo "STAGE_NAME=Functional on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV + echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV + echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV + echo "DISTRO=${DISTRO_NAME_UPPER}_$DISTRO_VERSION_MAJOR" >> $GITHUB_ENV + echo -n "PROVISION_DISTRO=${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}" >> $GITHUB_ENV + echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ + $GITHUB_ENV + echo "LABEL=$LABEL" >> $GITHUB_ENV + echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 500 + ref: ${{ github.event.pull_request.head.sha }} + - name: Request and Provision a Cluster + timeout-minutes: 7200 + uses: ./.github/actions/provision-cluster + with: + condition: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && \ + env.CP_SKIP_FUNC_TEST != 'true' + - name: Run Test + timeout-minutes: 7200 + if: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && env.CP_SKIP_FUNC_TEST != 'true' + id: run-test + run: | + . ci/gha_functions.sh + NODE_COUNT="$NODE_COUNT" + TEST_TAG={{ $inputs.tags }} + FTEST_ARG="${{ env.FTEST_ARG }}" ci/functional/test_main.sh + - name: Cancel cluster request (if cancelled after requesting) + if: cancelled() + run: | + set -eux + . ci/gha_functions.sh + if ! JENKINS_URL="${{ env.JENKINS_URL }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ + cancel_provision; then + # probably already provisioned and needs unprovisioning + if ! cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}"; then + exit 1 + fi + fi + - name: Job cleanup + if: (!cancelled() && (success() || failure())) + run: | + set -eux + . ci/gha_functions.sh + NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh || true + cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}" + - name: Publish test results + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + check_name: ${{ env.STAGE_NAME }} Test Results (old) + github_token: ${{ secrets.GITHUB_TOKEN }} + junit_files: ${{ env.STAGE_NAME }}/**/results.xml + - name: Publish artifacts + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} artifacts + path: ${{ env.STAGE_NAME }}/** + - name: Upload test results + if: (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} test-results + path: ${{ env.STAGE_NAME }}/**/results.xml + - name: Update commit status + uses: ouzi-dev/commit-status-updater@v2 + with: + # yamllint disable-line rule:line-length + name: 'test/Functional on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' + status: "${{ job.status }}" diff --git a/.github/workflows/code-coverage-test.yml b/.github/workflows/code-coverage-test.yml index 804ef1d6b1b..ab5d3245d21 100644 --- a/.github/workflows/code-coverage-test.yml +++ b/.github/workflows/code-coverage-test.yml @@ -1,17 +1,5 @@ name: Code Coverage Test -env: - # TODO: we really need to define a list of supported versions (ideally it's no more than 2) - # build is done on the lowest version and test on the highest with a "sanity test" - # stage done on all versions in the list except the highest - EL8_BUILD_VERSION: 8.6 - EL8_VERSION: 8.8 - EL9_BUILD_VERSION: 9 - EL9_VERSION: 9 - LEAP15_VERSION: 15.5 - - JENKINS_URL: https://build.hpdd.intel.com/ - on: pull_request: # Remove after testing schedule: @@ -28,391 +16,19 @@ defaults: permissions: {} jobs: - - Build-RPM: - name: Bullseye Build RPM - permissions: - statuses: write - runs-on: [self-hosted, docker] - if: ((!cancelled()) || success() || failure()) - strategy: - matrix: - distro: ["el8"] - fail-fast: false - env: - ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ - DAOS_EMAIL: phil.henderson@intel.com - DAOS_FULLNAME: daos-stack - DISTRO: ${{ matrix.distro }} - DISTRO_REPOS: disabled - # DOCKER_BUILDKIT: 0 - ARTIFACTS_URL: file:///scratch/job_repos/ - MOCK_OPTIONS: --uniqueext=${{ github.run_id }} - PR_NUM: ${{ github.event.pull_request.number }} - # TODO -- this should be on stable, backed up storage, not /scratch - # yamllint disable-line rule:line-length - REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ github.event.pull_request.number }}/ - REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ - RUN_ID: ${{ github.run_id }} - TARGET: ${{ matrix.distro }} - # keep VS Code's GHA linting happy - STAGE_NAME: - DISTRO_NAME: - DISTRO_VERSION: - CP_LEAP15_VERSION: - COMMIT_STATUS_DISTRO_VERSION: - FVERSION: - steps: - # - name: Import commit pragmas - # uses: ./.github/actions/import-commit-pragmas - - name: Set variables - run: | - FVERSION="38" - case ${{ matrix.distro }} in - 'el8') - CHROOT_NAME="rocky+epel-8-x86_64" - DISTRO_NAME="EL" - DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}" - COMMIT_STATUS_DISTRO_VERSION="8" - ;; - 'el9') - CHROOT_NAME="rocky+epel-9-x86_64" - DISTRO_NAME="EL" - DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}" - ;; - 'leap15') - CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || - env.LEAP15_VERSION }}-x86_64" - DISTRO_NAME="Leap" - DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" - ;; - esac - echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV - echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV - echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV - echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV - echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV - echo "FVERSION=$FVERSION" >> $GITHUB_ENV - echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV - echo "COVFN_DISABLED=false" >> $GITHUB_ENV - echo "BULLSEYE=yes" >> $GITHUB_ENV - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Build RPM Docker image - id: build-rpm-docker-image - continue-on-error: true - run: docker build --file utils/rpms/packaging/Dockerfile.mockbuild - --build-arg CACHEBUST=$(date +%s%3N) - --build-arg CB0=$(date +%V) - --build-arg REPO_FILE_URL=$REPO_FILE_URL - --build-arg UID=$(id -u) - --build-arg FVERSION=${{ env.FVERSION }} - --build-arg JENKINS_URL=${{ env.JENKINS_URL }} - --build-arg BULLSEYE_KEY=${{ secrets.BULLSEYE_LICENSE_KEY }} - --tag mock-build - utils/rpms - - name: Build RPM - id: build-rpm - continue-on-error: true - # yamllint disable rule:line-length - run: rm -rf mock_result; - mkdir -p mock_result; - docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }} - --user build - -v "$PWD":"$PWD" -w "$PWD" - -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result - --privileged=true - -e DAOS_FULLNAME="$DAOS_FULLNAME" - -e DAOS_EMAIL="$DAOS_EMAIL" - -e DISTRO_VERSION="$DISTRO_VERSION" - -e STAGE_NAME="$STAGE_NAME" - -e CHROOT_NAME="$CHROOT_NAME" - -e ARTIFACTORY_URL="$ARTIFACTORY_URL" - -e REPO_FILE_URL="$REPO_FILE_URL" - -e JENKINS_URL=${{ env.JENKINS_URL }} - -e TARGET="$TARGET" - -e COVFN_DISABLED="$COVFN_DISABLED" - mock-build ci/rpm/build.sh - # yamllint enable rule:line-length - - name: Build RPM failure log - id: build-rpm-fail-log - continue-on-error: true - if: steps.build-rpm.outcome != 'success' - run: cat mock_result/root.log; - cat mock_result/build.log - - name: Save RPM build logs - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: ${{ env.STAGE_NAME }} logs - path: | - mock_result/root.log - mock_result/build.log - - name: Create lastBuild and lastSuccessfulBuild symlinks - if: steps.build-rpm.outcome == 'success' - run: . ci/gha_functions.sh; - mkdir -p ${REPO_PATH}; - rm -f ${REPO_PATH}last{,Successful}Build; - ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild; - ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild - - name: Create repo - id: create-repo - if: steps.build-rpm.outcome == 'success' - continue-on-error: true - run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh - - name: Test repo - id: test-repo - if: steps.create-repo.outcome == 'success' - continue-on-error: true - run: . ci/gha_functions.sh; - dnf --disablerepo=\* --repofrompath - testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET - repoquery -a - - name: Remove lastSuccessfulBuild link and exit failure - if: steps.test-repo.outcome != 'success' - run: rm -f ${REPO_PATH}lastSuccessfulBuild; - exit 1 - - name: Publish RPMs - uses: actions/upload-artifact@v4 - with: - name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} RPM repository - path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} - - name: Update commit status - uses: ouzi-dev/commit-status-updater@v2 - with: - # yamllint disable-line rule:line-length - name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' - status: "${{ job.status }}" - - # Unit-Test: - # name: Bullseye Unit Test - # runs-on: [self-hosted, docker] - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - # with: - # submodules: true - # ref: ${{ github.event.pull_request.head.sha }} - # - name: Build deps in Docker - # run: docker build . --file utils/docker/Dockerfile.el.8 - # --build-arg DAOS_BUILD=no - # --build-arg DEPS_JOBS=50 - # - name: Build daos in Docker - # run: docker build . --file utils/docker/Dockerfile.el.8 - # --build-arg DEPS_JOBS=50 - # --build-arg DAOS_KEEP_SRC=yes - # --build-arg DAOS_TARGET_TYPE=debug - # --build-arg DAOS_JAVA_BUILD=yes - # --build-arg JENKINS_URL=${{ env.JENKINS_URL }} - # --build-arg BULLSEYE=${{ secrets.BULLSEYE_LICENSE_KEY }} - # --tag build-image-${{github.run_id}}-${{github.run_attempt}} - # - name: Run Unit Testing - # run: docker exec -t build-image-${{github.run_id}}-${{github.run_attempt}} - # sh -c "ls -al"; - # docker run --name unit-test-${{github.run_id}}-${{github.run_attempt}} - # -e STAGE_NAME="Unit Test Bullseye" - # -e BULLSEYE=${{ secrets.BULLSEYE_LICENSE_KEY }} - # build-image-${{github.run_id}}-${{github.run_attempt}} - # ci/unit/test_main_node.sh - # - name: Fetch test results - # run: docker cp unit-test-${{github.run_id}}-${{github.run_attempt}}:/covc_test_logs - # ./artifacts; - # docker cp unit-test-${{github.run_id}}-${{github.run_attempt}}:/covc_vm_test - # ./artifacts - # - name: Upload test results - # uses: actions/upload-artifact@v4 - # with: - # name: Bullseye Unit Testing test-results - # path: ./artifacts/ - - Functional-Test: - name: Bullseye Functional Test - runs-on: [self-hosted, wolf] - permissions: - statuses: write - # https://github.com/EnricoMi/publish-unit-test-result-action#permissions - checks: write - pull-requests: write - timeout-minutes: 7200 - needs: [Build-RPM] - strategy: - matrix: - distro: ["el8"] - fail-fast: false - # https://github.com/actions/runner/issues/491#issuecomment-926924523 + call-build-rpm: + uses: ./.github/actions/build-rpm + with: + distro: el8 + covfn_disabled: false + + call-functional-test: + uses: ./.github/actions/functional-test + needs: [call-build-rpm] if: | (!cancelled()) && - (needs.Build-RPM.result == 'success' || - needs.Build-RPM.result == 'skipped') - env: - CONFIG_POWER_ONLY: false - PRAGMA_SUFFIX: -vm - OPERATIONS_EMAIL: phil.henderson@intel.com - TEST_RPMS: true - COMMIT_MESSAGE: - JENKINS_URL: https://build.hpdd.intel.com/ - REPOSITORY_URL: https://repo.dc.hpdd.intel.com/ - REMOVE_EXISTING_RPMS: false - # TODO -- this should be on stable, backedup storage - ARTIFACTS_URL: file:///scratch/job_repos/ - REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ - # keep VS Code's GHA linting happy - NODESTRING: - CP_PR_REPOS: - CP_FEATURES: - CP_TEST_TAG: - CP_EL8_VM9_LABEL: - CP_EL9_VM9_LABEL: - CP_LEAP15_VM9_LABEL: - CP_PRIORITY: - CP_EL8_VERSION: - CP_EL9_VERSION: - CP_LEAP15_VERSION: - DISTRO: - CLUSTER_REQUEST_reqid: - STAGE_NAME: - QUEUE_URL: - LABEL: - DISTRO_NAME: - DISTRO_VERSION: - COMMIT_STATUS_DISTRO_VERSION: - FTEST_ARG: - steps: - # - name: Import commit pragmas - # uses: ./.github/actions/import-commit-pragmas - - name: Set variables - run: | - set -eux - env - STAGE_TAGS="vm" - FTEST_ARG="--nvme=auto" - INST_RPMS="daos-client daos-tests daos-server daos-serialize daos-tests-internal" - case "${{ matrix.distro }}" in - 'el8') - CHROOT_NAME="rocky+epel-8-x86_64" - DISTRO_NAME="EL" - DISTRO_NAME_UPPER="EL" - DISTRO_NAME_LOWER="el" - DISTRO_VERSION="${{ env.CP_EL8_VERSION && - env.CP_EL8_VERSION || env.EL8_VERSION }}" - DISTRO_VERSION_MAJOR="8" - OPENMPI="openmpi" - LABEL="${{ env.CP_EL8_VM9_LABEL && - env.CP_EL8_VM9_LABEL || 'ci_vm9' }}" - ;; - 'el9') - CHROOT_NAME="rocky+epel-9-x86_64" - DISTRO_NAME="EL" - DISTRO_NAME_UPPER="EL" - DISTRO_NAME_LOWER="el" - DISTRO_VERSION="${{ env.CP_EL9_VERSION && - env.CP_EL9_VERSION || env.EL9_VERSION }}" - DISTRO_VERSION_MAJOR="9" - PROV_DISTRO_VERSION_MAJOR="8" - OPENMPI="openmpi" - LABEL="${{ env.CP_EL9_VM9_LABEL && - env.CP_EL9_VM9_LABEL || 'ci_vm9' }}" - ;; - 'leap15') - CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || - env.LEAP15_VERSION }}-x86_64" - DISTRO_NAME="Leap" - DISTRO_NAME_UPPER="LEAP" - DISTRO_NAME_LOWER="leap" - DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" - DISTRO_VERSION_MAJOR="15" - OPENMPI="openmpi3" - LABEL="${{ env.CP_LEAP15_VM9_LABEL && - env.CP_LEAP15_VM9_LABEL || 'ci_vm9' }}" - ;; - esac - echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV - echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV - echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV - echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV - echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV - echo "STAGE_NAME=Functional on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV - echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV - echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV - echo "DISTRO=${DISTRO_NAME_UPPER}_$DISTRO_VERSION_MAJOR" >> $GITHUB_ENV - echo -n "PROVISION_DISTRO=${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV - echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}" >> $GITHUB_ENV - echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV - echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ - $GITHUB_ENV - echo "LABEL=$LABEL" >> $GITHUB_ENV - echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 500 - ref: ${{ github.event.pull_request.head.sha }} - - name: Request and Provision a Cluster - timeout-minutes: 7200 - uses: ./.github/actions/provision-cluster - with: - condition: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && \ - env.CP_SKIP_FUNC_TEST != 'true' - - name: Run Test - timeout-minutes: 7200 - if: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && env.CP_SKIP_FUNC_TEST != 'true' - id: run-test - run: | - . ci/gha_functions.sh - NODE_COUNT="$NODE_COUNT" \ - TEST_TAG="pr,vm" \ - FTEST_ARG="${{ env.FTEST_ARG }}" ci/functional/test_main.sh - - name: Cancel cluster request (if cancelled after requesting) - if: cancelled() - run: | - set -eux - . ci/gha_functions.sh - if ! JENKINS_URL="${{ env.JENKINS_URL }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ - cancel_provision; then - # probably already provisioned and needs unprovisioning - if ! cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}"; then - exit 1 - fi - fi - - name: Job cleanup - if: (!cancelled() && (success() || failure())) - run: | - set -eux - . ci/gha_functions.sh - NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh || true - cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}" - - name: Publish test results - if: (!cancelled()) && (success() || failure()) && - steps.run-test.outcome != 'skipped' - uses: EnricoMi/publish-unit-test-result-action@v2 - with: - check_name: ${{ env.STAGE_NAME }} Test Results (old) - github_token: ${{ secrets.GITHUB_TOKEN }} - junit_files: ${{ env.STAGE_NAME }}/**/results.xml - - name: Publish artifacts - if: (!cancelled()) && (success() || failure()) && - steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 - with: - name: ${{ env.STAGE_NAME }} artifacts - path: ${{ env.STAGE_NAME }}/** - - name: Upload test results - if: (success() || failure()) && - steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 - with: - name: ${{ env.STAGE_NAME }} test-results - path: ${{ env.STAGE_NAME }}/**/results.xml - - name: Update commit status - uses: ouzi-dev/commit-status-updater@v2 - with: - # yamllint disable-line rule:line-length - name: 'test/Functional on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' - status: "${{ job.status }}" + (needs.call-build-rpm.result == 'success' || + needs.call-build-rpm.result == 'skipped') + with: + distro: el8 + tags: test_one_pool_vm From 94163e2d86a1293a435fb94ed11a201f9fa9d759 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 11 Oct 2024 17:23:17 -0400 Subject: [PATCH 03/16] Updates. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/build-rpm/action.yml | 315 ++++++++++--------- .github/actions/functional-test/action.yml | 341 ++++++++++----------- .github/workflows/code-coverage-test.yml | 46 ++- 3 files changed, 353 insertions(+), 349 deletions(-) diff --git a/.github/actions/build-rpm/action.yml b/.github/actions/build-rpm/action.yml index efdab10d070..23da3f615d7 100644 --- a/.github/actions/build-rpm/action.yml +++ b/.github/actions/build-rpm/action.yml @@ -1,16 +1,6 @@ name: 'Build RPM' description: 'Build DAOS RPMs' -env: - # TODO: we really need to define a list of supported versions (ideally it's no more than 2) - # build is done on the lowest version and test on the highest with a "sanity test" - # stage done on all versions in the list except the highest - EL8_BUILD_VERSION: 8.6 - EL8_VERSION: 8.8 - EL9_BUILD_VERSION: 9 - EL9_VERSION: 9 - LEAP15_VERSION: 15.5 - on: workflow_call: inputs: @@ -24,155 +14,158 @@ on: type: bool default: true -jobs: - Build-RPM: - name: Bullseye Build RPM - permissions: - statuses: write - runs-on: [self-hosted, docker] - if: ((!cancelled()) || success() || failure()) - env: - ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ - JENKINS_URL: https://build.hpdd.intel.com/ - DAOS_EMAIL: phil.henderson@intel.com - DAOS_FULLNAME: daos-stack - DISTRO: ${{ inputs.distro }} - DISTRO_REPOS: disabled - # DOCKER_BUILDKIT: 0 - ARTIFACTS_URL: file:///scratch/job_repos/ - MOCK_OPTIONS: --uniqueext=${{ github.run_id }} - PR_NUM: ${{ github.event.pull_request.number }} - # TODO -- this should be on stable, backed up storage, not /scratch +env: + # TODO: we really need to define a list of supported versions (ideally it's no more than 2) + # build is done on the lowest version and test on the highest with a "sanity test" + # stage done on all versions in the list except the highest + EL8_BUILD_VERSION: 8.6 + EL8_VERSION: 8.8 + EL9_BUILD_VERSION: 9 + EL9_VERSION: 9 + LEAP15_VERSION: 15.5 + + ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ + JENKINS_URL: https://build.hpdd.intel.com/ + DAOS_EMAIL: phil.henderson@intel.com + DAOS_FULLNAME: daos-stack + DISTRO: ${{ inputs.distro }} + DISTRO_REPOS: disabled + # DOCKER_BUILDKIT: 0 + ARTIFACTS_URL: file:///scratch/job_repos/ + MOCK_OPTIONS: --uniqueext=${{ github.run_id }} + PR_NUM: ${{ github.event.pull_request.number }} + # TODO -- this should be on stable, backed up storage, not /scratch + # yamllint disable-line rule:line-length + REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ github.event.pull_request.number }}/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + RUN_ID: ${{ github.run_id }} + TARGET: ${{ inputs.distro }} + # keep VS Code's GHA linting happy + STAGE_NAME: + DISTRO_NAME: + DISTRO_VERSION: + CP_LEAP15_VERSION: + COMMIT_STATUS_DISTRO_VERSION: + FVERSION: + +runs: + - name: Set variables + run: | + FVERSION="38" + case ${{ inputs.distro }} in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}" + COMMIT_STATUS_DISTRO_VERSION="8" + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}" + ;; + 'leap15') + CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || + env.LEAP15_VERSION }}-x86_64" + DISTRO_NAME="Leap" + DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" + ;; + esac + echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV + echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV + echo "FVERSION=$FVERSION" >> $GITHUB_ENV + echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV + echo "COVFN_DISABLED=${{ inputs.covfn_disabled }}" >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@v4.1.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Build RPM Docker image + id: build-rpm-docker-image + continue-on-error: true + run: docker build --file utils/rpms/packaging/Dockerfile.mockbuild + --build-arg CACHEBUST=$(date +%s%3N) + --build-arg CB0=$(date +%V) + --build-arg REPO_FILE_URL=$REPO_FILE_URL + --build-arg UID=$(id -u) + --build-arg FVERSION=${{ env.FVERSION }} + --build-arg JENKINS_URL=${{ env.JENKINS_URL }} + --tag mock-build + utils/rpms + - name: Build RPM + id: build-rpm + continue-on-error: true + # yamllint disable rule:line-length + run: rm -rf mock_result; + mkdir -p mock_result; + docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.distro }} + --user build + -v "$PWD":"$PWD" -w "$PWD" + -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result + --privileged=true + -e DAOS_FULLNAME="$DAOS_FULLNAME" + -e DAOS_EMAIL="$DAOS_EMAIL" + -e DISTRO_VERSION="$DISTRO_VERSION" + -e STAGE_NAME="$STAGE_NAME" + -e CHROOT_NAME="$CHROOT_NAME" + -e ARTIFACTORY_URL="$ARTIFACTORY_URL" + -e REPO_FILE_URL="$REPO_FILE_URL" + -e JENKINS_URL=${{ env.JENKINS_URL }} + -e TARGET="$TARGET" + -e COVFN_DISABLED="$COVFN_DISABLED" + mock-build ci/rpm/build.sh + # yamllint enable rule:line-length + - name: Build RPM failure log + id: build-rpm-fail-log + continue-on-error: true + if: steps.build-rpm.outcome != 'success' + run: cat mock_result/root.log; + cat mock_result/build.log + - name: Save RPM build logs + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} logs + path: | + mock_result/root.log + mock_result/build.log + - name: Create lastBuild and lastSuccessfulBuild symlinks + if: steps.build-rpm.outcome == 'success' + run: . ci/gha_functions.sh; + mkdir -p ${REPO_PATH}; + rm -f ${REPO_PATH}last{,Successful}Build; + ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild; + ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild + - name: Create repo + id: create-repo + if: steps.build-rpm.outcome == 'success' + continue-on-error: true + run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh + - name: Test repo + id: test-repo + if: steps.create-repo.outcome == 'success' + continue-on-error: true + run: . ci/gha_functions.sh; + dnf --disablerepo=\* --repofrompath + testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET + repoquery -a + - name: Remove lastSuccessfulBuild link and exit failure + if: steps.test-repo.outcome != 'success' + run: rm -f ${REPO_PATH}lastSuccessfulBuild; + exit 1 + - name: Publish RPMs + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} RPM repository + path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} + - name: Update commit status + uses: ouzi-dev/commit-status-updater@v2 + with: # yamllint disable-line rule:line-length - REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ github.event.pull_request.number }}/ - REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ - RUN_ID: ${{ github.run_id }} - TARGET: ${{ inputs.distro }} - # keep VS Code's GHA linting happy - STAGE_NAME: - DISTRO_NAME: - DISTRO_VERSION: - CP_LEAP15_VERSION: - COMMIT_STATUS_DISTRO_VERSION: - FVERSION: - steps: - - name: Set variables - run: | - FVERSION="38" - case ${{ inputs.distro }} in - 'el8') - CHROOT_NAME="rocky+epel-8-x86_64" - DISTRO_NAME="EL" - DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}" - COMMIT_STATUS_DISTRO_VERSION="8" - ;; - 'el9') - CHROOT_NAME="rocky+epel-9-x86_64" - DISTRO_NAME="EL" - DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}" - ;; - 'leap15') - CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || - env.LEAP15_VERSION }}-x86_64" - DISTRO_NAME="Leap" - DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" - ;; - esac - echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV - echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV - echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV - echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV - echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV - echo "FVERSION=$FVERSION" >> $GITHUB_ENV - echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV - echo "COVFN_DISABLED=${{ inputs.covfn_disabled }}" >> $GITHUB_ENV - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Build RPM Docker image - id: build-rpm-docker-image - continue-on-error: true - run: docker build --file utils/rpms/packaging/Dockerfile.mockbuild - --build-arg CACHEBUST=$(date +%s%3N) - --build-arg CB0=$(date +%V) - --build-arg REPO_FILE_URL=$REPO_FILE_URL - --build-arg UID=$(id -u) - --build-arg FVERSION=${{ env.FVERSION }} - --build-arg JENKINS_URL=${{ env.JENKINS_URL }} - --tag mock-build - utils/rpms - - name: Build RPM - id: build-rpm - continue-on-error: true - # yamllint disable rule:line-length - run: rm -rf mock_result; - mkdir -p mock_result; - docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.distro }} - --user build - -v "$PWD":"$PWD" -w "$PWD" - -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result - --privileged=true - -e DAOS_FULLNAME="$DAOS_FULLNAME" - -e DAOS_EMAIL="$DAOS_EMAIL" - -e DISTRO_VERSION="$DISTRO_VERSION" - -e STAGE_NAME="$STAGE_NAME" - -e CHROOT_NAME="$CHROOT_NAME" - -e ARTIFACTORY_URL="$ARTIFACTORY_URL" - -e REPO_FILE_URL="$REPO_FILE_URL" - -e JENKINS_URL=${{ env.JENKINS_URL }} - -e TARGET="$TARGET" - -e COVFN_DISABLED="$COVFN_DISABLED" - mock-build ci/rpm/build.sh - # yamllint enable rule:line-length - - name: Build RPM failure log - id: build-rpm-fail-log - continue-on-error: true - if: steps.build-rpm.outcome != 'success' - run: cat mock_result/root.log; - cat mock_result/build.log - - name: Save RPM build logs - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: ${{ env.STAGE_NAME }} logs - path: | - mock_result/root.log - mock_result/build.log - - name: Create lastBuild and lastSuccessfulBuild symlinks - if: steps.build-rpm.outcome == 'success' - run: . ci/gha_functions.sh; - mkdir -p ${REPO_PATH}; - rm -f ${REPO_PATH}last{,Successful}Build; - ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild; - ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild - - name: Create repo - id: create-repo - if: steps.build-rpm.outcome == 'success' - continue-on-error: true - run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh - - name: Test repo - id: test-repo - if: steps.create-repo.outcome == 'success' - continue-on-error: true - run: . ci/gha_functions.sh; - dnf --disablerepo=\* --repofrompath - testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET - repoquery -a - - name: Remove lastSuccessfulBuild link and exit failure - if: steps.test-repo.outcome != 'success' - run: rm -f ${REPO_PATH}lastSuccessfulBuild; - exit 1 - - name: Publish RPMs - uses: actions/upload-artifact@v4 - with: - name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} RPM repository - path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} - - name: Update commit status - uses: ouzi-dev/commit-status-updater@v2 - with: - # yamllint disable-line rule:line-length - name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' - status: "${{ job.status }}" + name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' + status: "${{ job.status }}" diff --git a/.github/actions/functional-test/action.yml b/.github/actions/functional-test/action.yml index 5ac043110e5..e5997064c0e 100644 --- a/.github/actions/functional-test/action.yml +++ b/.github/actions/functional-test/action.yml @@ -17,179 +17,170 @@ on: type: string default: --nvme=auto -jobs: - runs-on: [self-hosted, wolf] - permissions: - statuses: write - # https://github.com/EnricoMi/publish-unit-test-result-action#permissions - checks: write - pull-requests: write - timeout-minutes: 7200 - env: - CONFIG_POWER_ONLY: false - PRAGMA_SUFFIX: -vm - OPERATIONS_EMAIL: phil.henderson@intel.com - TEST_RPMS: true - COMMIT_MESSAGE: - JENKINS_URL: https://build.hpdd.intel.com/ - REPOSITORY_URL: https://repo.dc.hpdd.intel.com/ - REMOVE_EXISTING_RPMS: false - # TODO -- this should be on stable, backedup storage - ARTIFACTS_URL: file:///scratch/job_repos/ - REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ - # keep VS Code's GHA linting happy - NODESTRING: - CP_PR_REPOS: - CP_FEATURES: - CP_TEST_TAG: - CP_EL8_VM9_LABEL: - CP_EL9_VM9_LABEL: - CP_LEAP15_VM9_LABEL: - CP_PRIORITY: - CP_EL8_VERSION: - CP_EL9_VERSION: - CP_LEAP15_VERSION: - DISTRO: - CLUSTER_REQUEST_reqid: - STAGE_NAME: - QUEUE_URL: - LABEL: - DISTRO_NAME: - DISTRO_VERSION: - COMMIT_STATUS_DISTRO_VERSION: - FTEST_ARG: - steps: - # - name: Import commit pragmas - # uses: ./.github/actions/import-commit-pragmas - - name: Set variables - run: | - set -eux - env - STAGE_TAGS="vm" - FTEST_ARG={{ $inputs.ftest_args }} - INST_RPMS="daos-client daos-tests daos-server daos-serialize daos-tests-internal" - case "${{ inputs.distro }}" in - 'el8') - CHROOT_NAME="rocky+epel-8-x86_64" - DISTRO_NAME="EL" - DISTRO_NAME_UPPER="EL" - DISTRO_NAME_LOWER="el" - DISTRO_VERSION="${{ env.CP_EL8_VERSION && - env.CP_EL8_VERSION || env.EL8_VERSION }}" - DISTRO_VERSION_MAJOR="8" - OPENMPI="openmpi" - LABEL="${{ env.CP_EL8_VM9_LABEL && - env.CP_EL8_VM9_LABEL || 'ci_vm9' }}" - ;; - 'el9') - CHROOT_NAME="rocky+epel-9-x86_64" - DISTRO_NAME="EL" - DISTRO_NAME_UPPER="EL" - DISTRO_NAME_LOWER="el" - DISTRO_VERSION="${{ env.CP_EL9_VERSION && - env.CP_EL9_VERSION || env.EL9_VERSION }}" - DISTRO_VERSION_MAJOR="9" - PROV_DISTRO_VERSION_MAJOR="8" - OPENMPI="openmpi" - LABEL="${{ env.CP_EL9_VM9_LABEL && - env.CP_EL9_VM9_LABEL || 'ci_vm9' }}" - ;; - 'leap15') - CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || - env.LEAP15_VERSION }}-x86_64" - DISTRO_NAME="Leap" - DISTRO_NAME_UPPER="LEAP" - DISTRO_NAME_LOWER="leap" - DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" - DISTRO_VERSION_MAJOR="15" - OPENMPI="openmpi3" - LABEL="${{ env.CP_LEAP15_VM9_LABEL && - env.CP_LEAP15_VM9_LABEL || 'ci_vm9' }}" - ;; - esac - echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV - echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV - echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV - echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV - echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV - echo "STAGE_NAME=Functional on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV - echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV - echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV - echo "DISTRO=${DISTRO_NAME_UPPER}_$DISTRO_VERSION_MAJOR" >> $GITHUB_ENV - echo -n "PROVISION_DISTRO=${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV - echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}" >> $GITHUB_ENV - echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV - echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ - $GITHUB_ENV - echo "LABEL=$LABEL" >> $GITHUB_ENV - echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 500 - ref: ${{ github.event.pull_request.head.sha }} - - name: Request and Provision a Cluster - timeout-minutes: 7200 - uses: ./.github/actions/provision-cluster - with: - condition: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && \ - env.CP_SKIP_FUNC_TEST != 'true' - - name: Run Test - timeout-minutes: 7200 - if: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && env.CP_SKIP_FUNC_TEST != 'true' - id: run-test - run: | - . ci/gha_functions.sh - NODE_COUNT="$NODE_COUNT" - TEST_TAG={{ $inputs.tags }} - FTEST_ARG="${{ env.FTEST_ARG }}" ci/functional/test_main.sh - - name: Cancel cluster request (if cancelled after requesting) - if: cancelled() - run: | - set -eux - . ci/gha_functions.sh - if ! JENKINS_URL="${{ env.JENKINS_URL }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ - cancel_provision; then - # probably already provisioned and needs unprovisioning - if ! cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}"; then - exit 1 - fi - fi - - name: Job cleanup - if: (!cancelled() && (success() || failure())) - run: | +env: + CONFIG_POWER_ONLY: false + PRAGMA_SUFFIX: -vm + OPERATIONS_EMAIL: phil.henderson@intel.com + TEST_RPMS: true + COMMIT_MESSAGE: + JENKINS_URL: https://build.hpdd.intel.com/ + REPOSITORY_URL: https://repo.dc.hpdd.intel.com/ + REMOVE_EXISTING_RPMS: false + # TODO -- this should be on stable, backedup storage + ARTIFACTS_URL: file:///scratch/job_repos/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + # keep VS Code's GHA linting happy + NODESTRING: + CP_PR_REPOS: + CP_FEATURES: + CP_TEST_TAG: + CP_EL8_VM9_LABEL: + CP_EL9_VM9_LABEL: + CP_LEAP15_VM9_LABEL: + CP_PRIORITY: + CP_EL8_VERSION: + CP_EL9_VERSION: + CP_LEAP15_VERSION: + DISTRO: + CLUSTER_REQUEST_reqid: + STAGE_NAME: + QUEUE_URL: + LABEL: + DISTRO_NAME: + DISTRO_VERSION: + COMMIT_STATUS_DISTRO_VERSION: + FTEST_ARG: + +runs: + - name: Set variables + run: | set -eux - . ci/gha_functions.sh - NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh || true - cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}" - - name: Publish test results - if: (!cancelled()) && (success() || failure()) && - steps.run-test.outcome != 'skipped' - uses: EnricoMi/publish-unit-test-result-action@v2 - with: - check_name: ${{ env.STAGE_NAME }} Test Results (old) - github_token: ${{ secrets.GITHUB_TOKEN }} - junit_files: ${{ env.STAGE_NAME }}/**/results.xml - - name: Publish artifacts - if: (!cancelled()) && (success() || failure()) && - steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 - with: - name: ${{ env.STAGE_NAME }} artifacts - path: ${{ env.STAGE_NAME }}/** - - name: Upload test results - if: (success() || failure()) && - steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 - with: - name: ${{ env.STAGE_NAME }} test-results - path: ${{ env.STAGE_NAME }}/**/results.xml - - name: Update commit status - uses: ouzi-dev/commit-status-updater@v2 - with: - # yamllint disable-line rule:line-length - name: 'test/Functional on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' - status: "${{ job.status }}" + env + STAGE_TAGS="vm" + FTEST_ARG={{ $inputs.ftest_args }} + INST_RPMS="daos-client daos-tests daos-server daos-serialize daos-tests-internal" + case "${{ inputs.distro }}" in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION="${{ env.CP_EL8_VERSION && + env.CP_EL8_VERSION || env.EL8_VERSION }}" + DISTRO_VERSION_MAJOR="8" + OPENMPI="openmpi" + LABEL="${{ env.CP_EL8_VM9_LABEL && + env.CP_EL8_VM9_LABEL || 'ci_vm9' }}" + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION="${{ env.CP_EL9_VERSION && + env.CP_EL9_VERSION || env.EL9_VERSION }}" + DISTRO_VERSION_MAJOR="9" + PROV_DISTRO_VERSION_MAJOR="8" + OPENMPI="openmpi" + LABEL="${{ env.CP_EL9_VM9_LABEL && + env.CP_EL9_VM9_LABEL || 'ci_vm9' }}" + ;; + 'leap15') + CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || + env.LEAP15_VERSION }}-x86_64" + DISTRO_NAME="Leap" + DISTRO_NAME_UPPER="LEAP" + DISTRO_NAME_LOWER="leap" + DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" + DISTRO_VERSION_MAJOR="15" + OPENMPI="openmpi3" + LABEL="${{ env.CP_LEAP15_VM9_LABEL && + env.CP_LEAP15_VM9_LABEL || 'ci_vm9' }}" + ;; + esac + echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV + echo "STAGE_NAME=Functional on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV + echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV + echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV + echo "DISTRO=${DISTRO_NAME_UPPER}_$DISTRO_VERSION_MAJOR" >> $GITHUB_ENV + echo -n "PROVISION_DISTRO=${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}" >> $GITHUB_ENV + echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ + $GITHUB_ENV + echo "LABEL=$LABEL" >> $GITHUB_ENV + echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 500 + ref: ${{ github.event.pull_request.head.sha }} + - name: Request and Provision a Cluster + timeout-minutes: 7200 + uses: ./.github/actions/provision-cluster + with: + condition: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && \ + env.CP_SKIP_FUNC_TEST != 'true' + - name: Run Test + timeout-minutes: 7200 + if: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && env.CP_SKIP_FUNC_TEST != 'true' + id: run-test + run: | + . ci/gha_functions.sh + NODE_COUNT="$NODE_COUNT" + TEST_TAG={{ $inputs.tags }} + FTEST_ARG="${{ env.FTEST_ARG }}" ci/functional/test_main.sh + - name: Cancel cluster request (if cancelled after requesting) + if: cancelled() + run: | + set -eux + . ci/gha_functions.sh + if ! JENKINS_URL="${{ env.JENKINS_URL }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ + cancel_provision; then + # probably already provisioned and needs un-provisioning + if ! cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}"; then + exit 1 + fi + fi + - name: Job cleanup + if: (!cancelled() && (success() || failure())) + run: | + set -eux + . ci/gha_functions.sh + NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh || true + cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}" + - name: Publish test results + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + check_name: ${{ env.STAGE_NAME }} Test Results (old) + github_token: ${{ secrets.GITHUB_TOKEN }} + junit_files: ${{ env.STAGE_NAME }}/**/results.xml + - name: Publish artifacts + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} artifacts + path: ${{ env.STAGE_NAME }}/** + - name: Upload test results + if: (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} test-results + path: ${{ env.STAGE_NAME }}/**/results.xml + - name: Update commit status + uses: ouzi-dev/commit-status-updater@v2 + with: + # yamllint disable-line rule:line-length + name: 'test/Functional on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' + status: "${{ job.status }}" diff --git a/.github/workflows/code-coverage-test.yml b/.github/workflows/code-coverage-test.yml index ab5d3245d21..7f80153f78e 100644 --- a/.github/workflows/code-coverage-test.yml +++ b/.github/workflows/code-coverage-test.yml @@ -16,19 +16,39 @@ defaults: permissions: {} jobs: - call-build-rpm: - uses: ./.github/actions/build-rpm - with: - distro: el8 - covfn_disabled: false + build-rpms: + name: Bullseye Build RPMs + runs-on: [self-hosted, docker] + permissions: + statuses: write + steps: + - name: Checkout code + uses: actions/checkout@v4.1.1 + - name: Build RPMs + uses: ./.github/actions/build-rpm + with: + distro: el8 + covfn_disabled: false - call-functional-test: - uses: ./.github/actions/functional-test - needs: [call-build-rpm] + functional-test: + name: Bullseye Functional Test + runs-on: [self-hosted, wolf] + permissions: + statuses: write + # https://github.com/EnricoMi/publish-unit-test-result-action#permissions + checks: write + pull-requests: write + timeout-minutes: 7200 + needs: [build-rpms] if: | (!cancelled()) && - (needs.call-build-rpm.result == 'success' || - needs.call-build-rpm.result == 'skipped') - with: - distro: el8 - tags: test_one_pool_vm + (needs.build-rpms.result == 'success' || + needs.build-rpms.result == 'skipped') + steps: + - name: Checkout code + uses: actions/checkout@v4.1.1 + - name: Build RPMs + uses: ./.github/actions/functional-test + with: + distro: el8 + tags: test_one_pool_vm From 007d69fcd92065c8a733db6b2b95ac3d80cc354e Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 11 Oct 2024 17:36:21 -0400 Subject: [PATCH 04/16] Build-rpm action updates Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/build-rpm/action.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/actions/build-rpm/action.yml b/.github/actions/build-rpm/action.yml index 23da3f615d7..707d08069e0 100644 --- a/.github/actions/build-rpm/action.yml +++ b/.github/actions/build-rpm/action.yml @@ -28,18 +28,10 @@ env: JENKINS_URL: https://build.hpdd.intel.com/ DAOS_EMAIL: phil.henderson@intel.com DAOS_FULLNAME: daos-stack - DISTRO: ${{ inputs.distro }} DISTRO_REPOS: disabled # DOCKER_BUILDKIT: 0 ARTIFACTS_URL: file:///scratch/job_repos/ - MOCK_OPTIONS: --uniqueext=${{ github.run_id }} - PR_NUM: ${{ github.event.pull_request.number }} - # TODO -- this should be on stable, backed up storage, not /scratch - # yamllint disable-line rule:line-length - REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ github.event.pull_request.number }}/ REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ - RUN_ID: ${{ github.run_id }} - TARGET: ${{ inputs.distro }} # keep VS Code's GHA linting happy STAGE_NAME: DISTRO_NAME: @@ -47,6 +39,11 @@ env: CP_LEAP15_VERSION: COMMIT_STATUS_DISTRO_VERSION: FVERSION: + PR_NUM: + MOCK_OPTIONS: + RUN_ID: + DISTRO: + TARGET: runs: - name: Set variables @@ -80,6 +77,13 @@ runs: echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV echo "FVERSION=$FVERSION" >> $GITHUB_ENV echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV + echo "PR_NUM=${{ github.event.pull_request.number }}" >> $GITHUB_ENV + # TODO -- this should be on stable, backed up storage, not /scratch + echo "REPO_PATH=/scratch/job_repos/daos-stack/job/daos/job/PR-${PR_NUM}/" >> $GITHUB_ENV + echo "MOCK_OPTIONS=--uniqueext=${{ github.run_id }}" >> $GITHUB_ENV + echo "RUN_ID=${{ github.run_id }}" >> $GITHUB_ENV + echo "DISTRO=${{ inputs.distro }}" >> $GITHUB_ENV + echo "TARGET=${{ inputs.distro }}" >> $GITHUB_ENV echo "COVFN_DISABLED=${{ inputs.covfn_disabled }}" >> $GITHUB_ENV - name: Checkout code uses: actions/checkout@v4.1.1 From a3d7efd1e142c62a8c0a6ee0846c4256d8961af7 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 11 Oct 2024 17:49:20 -0400 Subject: [PATCH 05/16] Fix actions syntax Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/build-rpm/action.yml | 293 ++++++++++----------- .github/actions/functional-test/action.yml | 263 +++++++++--------- 2 files changed, 277 insertions(+), 279 deletions(-) diff --git a/.github/actions/build-rpm/action.yml b/.github/actions/build-rpm/action.yml index 707d08069e0..499b130ed94 100644 --- a/.github/actions/build-rpm/action.yml +++ b/.github/actions/build-rpm/action.yml @@ -24,152 +24,149 @@ env: EL9_VERSION: 9 LEAP15_VERSION: 15.5 - ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ - JENKINS_URL: https://build.hpdd.intel.com/ - DAOS_EMAIL: phil.henderson@intel.com - DAOS_FULLNAME: daos-stack - DISTRO_REPOS: disabled - # DOCKER_BUILDKIT: 0 - ARTIFACTS_URL: file:///scratch/job_repos/ - REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ - # keep VS Code's GHA linting happy - STAGE_NAME: - DISTRO_NAME: - DISTRO_VERSION: - CP_LEAP15_VERSION: - COMMIT_STATUS_DISTRO_VERSION: - FVERSION: - PR_NUM: - MOCK_OPTIONS: - RUN_ID: - DISTRO: - TARGET: - runs: - - name: Set variables - run: | - FVERSION="38" - case ${{ inputs.distro }} in - 'el8') - CHROOT_NAME="rocky+epel-8-x86_64" - DISTRO_NAME="EL" - DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}" - COMMIT_STATUS_DISTRO_VERSION="8" - ;; - 'el9') - CHROOT_NAME="rocky+epel-9-x86_64" - DISTRO_NAME="EL" - DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}" - ;; - 'leap15') - CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || - env.LEAP15_VERSION }}-x86_64" - DISTRO_NAME="Leap" - DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" - ;; - esac - echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV - echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV - echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV - echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV - echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV - echo "FVERSION=$FVERSION" >> $GITHUB_ENV - echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV - echo "PR_NUM=${{ github.event.pull_request.number }}" >> $GITHUB_ENV - # TODO -- this should be on stable, backed up storage, not /scratch - echo "REPO_PATH=/scratch/job_repos/daos-stack/job/daos/job/PR-${PR_NUM}/" >> $GITHUB_ENV - echo "MOCK_OPTIONS=--uniqueext=${{ github.run_id }}" >> $GITHUB_ENV - echo "RUN_ID=${{ github.run_id }}" >> $GITHUB_ENV - echo "DISTRO=${{ inputs.distro }}" >> $GITHUB_ENV - echo "TARGET=${{ inputs.distro }}" >> $GITHUB_ENV - echo "COVFN_DISABLED=${{ inputs.covfn_disabled }}" >> $GITHUB_ENV - - name: Checkout code - uses: actions/checkout@v4.1.1 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Build RPM Docker image - id: build-rpm-docker-image - continue-on-error: true - run: docker build --file utils/rpms/packaging/Dockerfile.mockbuild - --build-arg CACHEBUST=$(date +%s%3N) - --build-arg CB0=$(date +%V) - --build-arg REPO_FILE_URL=$REPO_FILE_URL - --build-arg UID=$(id -u) - --build-arg FVERSION=${{ env.FVERSION }} - --build-arg JENKINS_URL=${{ env.JENKINS_URL }} - --tag mock-build - utils/rpms - - name: Build RPM - id: build-rpm - continue-on-error: true - # yamllint disable rule:line-length - run: rm -rf mock_result; - mkdir -p mock_result; - docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.distro }} - --user build - -v "$PWD":"$PWD" -w "$PWD" - -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result - --privileged=true - -e DAOS_FULLNAME="$DAOS_FULLNAME" - -e DAOS_EMAIL="$DAOS_EMAIL" - -e DISTRO_VERSION="$DISTRO_VERSION" - -e STAGE_NAME="$STAGE_NAME" - -e CHROOT_NAME="$CHROOT_NAME" - -e ARTIFACTORY_URL="$ARTIFACTORY_URL" - -e REPO_FILE_URL="$REPO_FILE_URL" - -e JENKINS_URL=${{ env.JENKINS_URL }} - -e TARGET="$TARGET" - -e COVFN_DISABLED="$COVFN_DISABLED" - mock-build ci/rpm/build.sh - # yamllint enable rule:line-length - - name: Build RPM failure log - id: build-rpm-fail-log - continue-on-error: true - if: steps.build-rpm.outcome != 'success' - run: cat mock_result/root.log; - cat mock_result/build.log - - name: Save RPM build logs - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: ${{ env.STAGE_NAME }} logs - path: | - mock_result/root.log - mock_result/build.log - - name: Create lastBuild and lastSuccessfulBuild symlinks - if: steps.build-rpm.outcome == 'success' - run: . ci/gha_functions.sh; - mkdir -p ${REPO_PATH}; - rm -f ${REPO_PATH}last{,Successful}Build; - ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild; - ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild - - name: Create repo - id: create-repo - if: steps.build-rpm.outcome == 'success' - continue-on-error: true - run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh - - name: Test repo - id: test-repo - if: steps.create-repo.outcome == 'success' - continue-on-error: true - run: . ci/gha_functions.sh; - dnf --disablerepo=\* --repofrompath - testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET - repoquery -a - - name: Remove lastSuccessfulBuild link and exit failure - if: steps.test-repo.outcome != 'success' - run: rm -f ${REPO_PATH}lastSuccessfulBuild; - exit 1 - - name: Publish RPMs - uses: actions/upload-artifact@v4 - with: - name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} RPM repository - path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} - - name: Update commit status - uses: ouzi-dev/commit-status-updater@v2 - with: - # yamllint disable-line rule:line-length - name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' - status: "${{ job.status }}" + env: + ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ + JENKINS_URL: https://build.hpdd.intel.com/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + DAOS_EMAIL: phil.henderson@intel.com + DAOS_FULLNAME: daos-stack + DISTRO_REPOS: disabled + DOCKER_BUILDKIT: 0 + ARTIFACTS_URL: file:///scratch/job_repos/ + MOCK_OPTIONS: --uniqueext=${{ github.run_id }} + PR_NUM: ${{ github.event.pull_request.number }} + # TODO -- this should be on stable, backedup storage, not /scratch + REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ env.PR_NUM }}/ + RUN_ID: ${{ github.run_id }} + DISTRO: ${{ inputs.distro }} + TARGET: ${{ inputs.distro }} + COVFN_DISABLED: ${{ inputs.covfn_disabled }} + # keep VS Code's GHA linting happy + STAGE_NAME: + DISTRO_NAME: + DISTRO_VERSION: + CP_LEAP15_VERSION: + COMMIT_STATUS_DISTRO_VERSION: + FVERSION: + + steps: + - name: Set variables + run: | + FVERSION="38" + case ${{ inputs.distro }} in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}" + COMMIT_STATUS_DISTRO_VERSION="8" + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}" + ;; + 'leap15') + CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || + env.LEAP15_VERSION }}-x86_64" + DISTRO_NAME="Leap" + DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" + ;; + esac + echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV + echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV + echo "FVERSION=$FVERSION" >> $GITHUB_ENV + echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@v4.1.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Build RPM Docker image + id: build-rpm-docker-image + continue-on-error: true + run: docker build --file utils/rpms/packaging/Dockerfile.mockbuild + --build-arg CACHEBUST=$(date +%s%3N) + --build-arg CB0=$(date +%V) + --build-arg REPO_FILE_URL=$REPO_FILE_URL + --build-arg UID=$(id -u) + --build-arg FVERSION=${{ env.FVERSION }} + --build-arg JENKINS_URL=${{ env.JENKINS_URL }} + --tag mock-build + utils/rpms + - name: Build RPM + id: build-rpm + continue-on-error: true + # yamllint disable rule:line-length + run: rm -rf mock_result; + mkdir -p mock_result; + docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.distro }} + --user build + -v "$PWD":"$PWD" -w "$PWD" + -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result + --privileged=true + -e DAOS_FULLNAME="$DAOS_FULLNAME" + -e DAOS_EMAIL="$DAOS_EMAIL" + -e DISTRO_VERSION="$DISTRO_VERSION" + -e STAGE_NAME="$STAGE_NAME" + -e CHROOT_NAME="$CHROOT_NAME" + -e ARTIFACTORY_URL="$ARTIFACTORY_URL" + -e REPO_FILE_URL="$REPO_FILE_URL" + -e JENKINS_URL=${{ env.JENKINS_URL }} + -e TARGET="$TARGET" + -e COVFN_DISABLED="$COVFN_DISABLED" + mock-build ci/rpm/build.sh + # yamllint enable rule:line-length + - name: Build RPM failure log + id: build-rpm-fail-log + continue-on-error: true + if: steps.build-rpm.outcome != 'success' + run: cat mock_result/root.log; + cat mock_result/build.log + - name: Save RPM build logs + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} logs + path: | + mock_result/root.log + mock_result/build.log + - name: Create lastBuild and lastSuccessfulBuild symlinks + if: steps.build-rpm.outcome == 'success' + run: . ci/gha_functions.sh; + mkdir -p ${REPO_PATH}; + rm -f ${REPO_PATH}last{,Successful}Build; + ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild; + ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild + - name: Create repo + id: create-repo + if: steps.build-rpm.outcome == 'success' + continue-on-error: true + run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh + - name: Test repo + id: test-repo + if: steps.create-repo.outcome == 'success' + continue-on-error: true + run: . ci/gha_functions.sh; + dnf --disablerepo=\* --repofrompath + testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET + repoquery -a + - name: Remove lastSuccessfulBuild link and exit failure + if: steps.test-repo.outcome != 'success' + run: rm -f ${REPO_PATH}lastSuccessfulBuild; + exit 1 + - name: Publish RPMs + uses: actions/upload-artifact@v4 + with: + name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} RPM repository + path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} + - name: Update commit status + uses: ouzi-dev/commit-status-updater@v2 + with: + # yamllint disable-line rule:line-length + name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' + status: "${{ job.status }}" diff --git a/.github/actions/functional-test/action.yml b/.github/actions/functional-test/action.yml index e5997064c0e..a20553f75cc 100644 --- a/.github/actions/functional-test/action.yml +++ b/.github/actions/functional-test/action.yml @@ -52,135 +52,136 @@ env: FTEST_ARG: runs: - - name: Set variables - run: | + steps: + - name: Set variables + run: | + set -eux + env + STAGE_TAGS="vm" + FTEST_ARG={{ $inputs.ftest_args }} + INST_RPMS="daos-client daos-tests daos-server daos-serialize daos-tests-internal" + case "${{ inputs.distro }}" in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION="${{ env.CP_EL8_VERSION && + env.CP_EL8_VERSION || env.EL8_VERSION }}" + DISTRO_VERSION_MAJOR="8" + OPENMPI="openmpi" + LABEL="${{ env.CP_EL8_VM9_LABEL && + env.CP_EL8_VM9_LABEL || 'ci_vm9' }}" + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION="${{ env.CP_EL9_VERSION && + env.CP_EL9_VERSION || env.EL9_VERSION }}" + DISTRO_VERSION_MAJOR="9" + PROV_DISTRO_VERSION_MAJOR="8" + OPENMPI="openmpi" + LABEL="${{ env.CP_EL9_VM9_LABEL && + env.CP_EL9_VM9_LABEL || 'ci_vm9' }}" + ;; + 'leap15') + CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || + env.LEAP15_VERSION }}-x86_64" + DISTRO_NAME="Leap" + DISTRO_NAME_UPPER="LEAP" + DISTRO_NAME_LOWER="leap" + DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && + env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" + DISTRO_VERSION_MAJOR="15" + OPENMPI="openmpi3" + LABEL="${{ env.CP_LEAP15_VM9_LABEL && + env.CP_LEAP15_VM9_LABEL || 'ci_vm9' }}" + ;; + esac + echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV + echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV + echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV + echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV + echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV + echo "STAGE_NAME=Functional on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV + echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV + echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV + echo "DISTRO=${DISTRO_NAME_UPPER}_$DISTRO_VERSION_MAJOR" >> $GITHUB_ENV + echo -n "PROVISION_DISTRO=${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}" >> $GITHUB_ENV + echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ + $GITHUB_ENV + echo "LABEL=$LABEL" >> $GITHUB_ENV + echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 500 + ref: ${{ github.event.pull_request.head.sha }} + - name: Request and Provision a Cluster + timeout-minutes: 7200 + uses: ./.github/actions/provision-cluster + with: + condition: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && \ + env.CP_SKIP_FUNC_TEST != 'true' + - name: Run Test + timeout-minutes: 7200 + if: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && env.CP_SKIP_FUNC_TEST != 'true' + id: run-test + run: | + . ci/gha_functions.sh + NODE_COUNT="$NODE_COUNT" + TEST_TAG={{ $inputs.tags }} + FTEST_ARG="${{ env.FTEST_ARG }}" ci/functional/test_main.sh + - name: Cancel cluster request (if cancelled after requesting) + if: cancelled() + run: | set -eux - env - STAGE_TAGS="vm" - FTEST_ARG={{ $inputs.ftest_args }} - INST_RPMS="daos-client daos-tests daos-server daos-serialize daos-tests-internal" - case "${{ inputs.distro }}" in - 'el8') - CHROOT_NAME="rocky+epel-8-x86_64" - DISTRO_NAME="EL" - DISTRO_NAME_UPPER="EL" - DISTRO_NAME_LOWER="el" - DISTRO_VERSION="${{ env.CP_EL8_VERSION && - env.CP_EL8_VERSION || env.EL8_VERSION }}" - DISTRO_VERSION_MAJOR="8" - OPENMPI="openmpi" - LABEL="${{ env.CP_EL8_VM9_LABEL && - env.CP_EL8_VM9_LABEL || 'ci_vm9' }}" - ;; - 'el9') - CHROOT_NAME="rocky+epel-9-x86_64" - DISTRO_NAME="EL" - DISTRO_NAME_UPPER="EL" - DISTRO_NAME_LOWER="el" - DISTRO_VERSION="${{ env.CP_EL9_VERSION && - env.CP_EL9_VERSION || env.EL9_VERSION }}" - DISTRO_VERSION_MAJOR="9" - PROV_DISTRO_VERSION_MAJOR="8" - OPENMPI="openmpi" - LABEL="${{ env.CP_EL9_VM9_LABEL && - env.CP_EL9_VM9_LABEL || 'ci_vm9' }}" - ;; - 'leap15') - CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || - env.LEAP15_VERSION }}-x86_64" - DISTRO_NAME="Leap" - DISTRO_NAME_UPPER="LEAP" - DISTRO_NAME_LOWER="leap" - DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" - DISTRO_VERSION_MAJOR="15" - OPENMPI="openmpi3" - LABEL="${{ env.CP_LEAP15_VM9_LABEL && - env.CP_LEAP15_VM9_LABEL || 'ci_vm9' }}" - ;; - esac - echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV - echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV - echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV - echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV - echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV - echo "STAGE_NAME=Functional on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV - echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV - echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV - echo "DISTRO=${DISTRO_NAME_UPPER}_$DISTRO_VERSION_MAJOR" >> $GITHUB_ENV - echo -n "PROVISION_DISTRO=${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV - echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}" >> $GITHUB_ENV - echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV - echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ - $GITHUB_ENV - echo "LABEL=$LABEL" >> $GITHUB_ENV - echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 500 - ref: ${{ github.event.pull_request.head.sha }} - - name: Request and Provision a Cluster - timeout-minutes: 7200 - uses: ./.github/actions/provision-cluster - with: - condition: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && \ - env.CP_SKIP_FUNC_TEST != 'true' - - name: Run Test - timeout-minutes: 7200 - if: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && env.CP_SKIP_FUNC_TEST != 'true' - id: run-test - run: | - . ci/gha_functions.sh - NODE_COUNT="$NODE_COUNT" - TEST_TAG={{ $inputs.tags }} - FTEST_ARG="${{ env.FTEST_ARG }}" ci/functional/test_main.sh - - name: Cancel cluster request (if cancelled after requesting) - if: cancelled() - run: | - set -eux - . ci/gha_functions.sh - if ! JENKINS_URL="${{ env.JENKINS_URL }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ - cancel_provision; then - # probably already provisioned and needs un-provisioning - if ! cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}"; then - exit 1 - fi - fi - - name: Job cleanup - if: (!cancelled() && (success() || failure())) - run: | - set -eux - . ci/gha_functions.sh - NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh || true - cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}" - - name: Publish test results - if: (!cancelled()) && (success() || failure()) && - steps.run-test.outcome != 'skipped' - uses: EnricoMi/publish-unit-test-result-action@v2 - with: - check_name: ${{ env.STAGE_NAME }} Test Results (old) - github_token: ${{ secrets.GITHUB_TOKEN }} - junit_files: ${{ env.STAGE_NAME }}/**/results.xml - - name: Publish artifacts - if: (!cancelled()) && (success() || failure()) && - steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 - with: - name: ${{ env.STAGE_NAME }} artifacts - path: ${{ env.STAGE_NAME }}/** - - name: Upload test results - if: (success() || failure()) && - steps.run-test.outcome != 'skipped' - uses: actions/upload-artifact@v4 - with: - name: ${{ env.STAGE_NAME }} test-results - path: ${{ env.STAGE_NAME }}/**/results.xml - - name: Update commit status - uses: ouzi-dev/commit-status-updater@v2 - with: - # yamllint disable-line rule:line-length - name: 'test/Functional on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' - status: "${{ job.status }}" + . ci/gha_functions.sh + if ! JENKINS_URL="${{ env.JENKINS_URL }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ + cancel_provision; then + # probably already provisioned and needs un-provisioning + if ! cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}"; then + exit 1 + fi + fi + - name: Job cleanup + if: (!cancelled() && (success() || failure())) + run: | + set -eux + . ci/gha_functions.sh + NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh || true + cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}" + - name: Publish test results + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + check_name: ${{ env.STAGE_NAME }} Test Results (old) + github_token: ${{ secrets.GITHUB_TOKEN }} + junit_files: ${{ env.STAGE_NAME }}/**/results.xml + - name: Publish artifacts + if: (!cancelled()) && (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} artifacts + path: ${{ env.STAGE_NAME }}/** + - name: Upload test results + if: (success() || failure()) && + steps.run-test.outcome != 'skipped' + uses: actions/upload-artifact@v4 + with: + name: ${{ env.STAGE_NAME }} test-results + path: ${{ env.STAGE_NAME }}/**/results.xml + - name: Update commit status + uses: ouzi-dev/commit-status-updater@v2 + with: + # yamllint disable-line rule:line-length + name: 'test/Functional on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' + status: "${{ job.status }}" From ae4bf4275b1c49a1762a42ed5e988470e0b7da07 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 11 Oct 2024 17:54:58 -0400 Subject: [PATCH 06/16] Using composite actions. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/build-rpm/action.yml | 6 ++++-- .github/actions/functional-test/action.yml | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-rpm/action.yml b/.github/actions/build-rpm/action.yml index 499b130ed94..faa67670f52 100644 --- a/.github/actions/build-rpm/action.yml +++ b/.github/actions/build-rpm/action.yml @@ -25,6 +25,7 @@ env: LEAP15_VERSION: 15.5 runs: + using: "composite" env: ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ JENKINS_URL: https://build.hpdd.intel.com/ @@ -36,8 +37,9 @@ runs: ARTIFACTS_URL: file:///scratch/job_repos/ MOCK_OPTIONS: --uniqueext=${{ github.run_id }} PR_NUM: ${{ github.event.pull_request.number }} - # TODO -- this should be on stable, backedup storage, not /scratch - REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ env.PR_NUM }}/ + # TODO -- this should be on stable, backed up storage, not /scratch + # yamllint disable-line rule:line-length + REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ github.event.pull_request.number }}/ RUN_ID: ${{ github.run_id }} DISTRO: ${{ inputs.distro }} TARGET: ${{ inputs.distro }} diff --git a/.github/actions/functional-test/action.yml b/.github/actions/functional-test/action.yml index a20553f75cc..9a4ca089f23 100644 --- a/.github/actions/functional-test/action.yml +++ b/.github/actions/functional-test/action.yml @@ -52,6 +52,7 @@ env: FTEST_ARG: runs: + using: "composite" steps: - name: Set variables run: | From f3d71a3716ec5619be7ab1a6d4d4894b1b74b9e5 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 17 Oct 2024 14:34:42 -0400 Subject: [PATCH 07/16] Attempt to resolve action errors. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/build-rpm/action.yml | 144 ++++++++------------- .github/actions/build-variables/action.yml | 113 ++++++++++++++++ .github/actions/functional-test/action.yml | 143 ++++++-------------- .github/workflows/code-coverage-test.yml | 36 ++++++ 4 files changed, 244 insertions(+), 192 deletions(-) create mode 100644 .github/actions/build-variables/action.yml diff --git a/.github/actions/build-rpm/action.yml b/.github/actions/build-rpm/action.yml index faa67670f52..7312e0b5670 100644 --- a/.github/actions/build-rpm/action.yml +++ b/.github/actions/build-rpm/action.yml @@ -10,80 +10,44 @@ on: - el8 - el9 - leap15 + version: + type: string + required: true + build_version: + type: string + required: true covfn_disabled: type: bool default: true - -env: - # TODO: we really need to define a list of supported versions (ideally it's no more than 2) - # build is done on the lowest version and test on the highest with a "sanity test" - # stage done on all versions in the list except the highest - EL8_BUILD_VERSION: 8.6 - EL8_VERSION: 8.8 - EL9_BUILD_VERSION: 9 - EL9_VERSION: 9 - LEAP15_VERSION: 15.5 + repo_file_url: + type: string + required: true + jenkins_url: + type: string + required: true + daos_full_name: + type: string + required: true + daos_email: + type: string + required: true + artifactory_url: + type: string + required: true + repo_path: + type: string + required: true runs: using: "composite" - env: - ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ - JENKINS_URL: https://build.hpdd.intel.com/ - REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ - DAOS_EMAIL: phil.henderson@intel.com - DAOS_FULLNAME: daos-stack - DISTRO_REPOS: disabled - DOCKER_BUILDKIT: 0 - ARTIFACTS_URL: file:///scratch/job_repos/ - MOCK_OPTIONS: --uniqueext=${{ github.run_id }} - PR_NUM: ${{ github.event.pull_request.number }} - # TODO -- this should be on stable, backed up storage, not /scratch - # yamllint disable-line rule:line-length - REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ github.event.pull_request.number }}/ - RUN_ID: ${{ github.run_id }} - DISTRO: ${{ inputs.distro }} - TARGET: ${{ inputs.distro }} - COVFN_DISABLED: ${{ inputs.covfn_disabled }} - # keep VS Code's GHA linting happy - STAGE_NAME: - DISTRO_NAME: - DISTRO_VERSION: - CP_LEAP15_VERSION: - COMMIT_STATUS_DISTRO_VERSION: - FVERSION: - steps: - - name: Set variables - run: | - FVERSION="38" - case ${{ inputs.distro }} in - 'el8') - CHROOT_NAME="rocky+epel-8-x86_64" - DISTRO_NAME="EL" - DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}" - COMMIT_STATUS_DISTRO_VERSION="8" - ;; - 'el9') - CHROOT_NAME="rocky+epel-9-x86_64" - DISTRO_NAME="EL" - DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}" - ;; - 'leap15') - CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || - env.LEAP15_VERSION }}-x86_64" - DISTRO_NAME="Leap" - DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" - ;; - esac - echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV - echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV - echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV - echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV - echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV - echo "FVERSION=$FVERSION" >> $GITHUB_ENV - echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV + - name: Set build variables + id: build-vars + uses: ./.github/actions/build-variables + with: + distro: ${{ inputs.distro }} + version: ${{ inputs.version }} + build_version: ${{ inputs.build_version }} - name: Checkout code uses: actions/checkout@v4.1.1 with: @@ -94,10 +58,10 @@ runs: run: docker build --file utils/rpms/packaging/Dockerfile.mockbuild --build-arg CACHEBUST=$(date +%s%3N) --build-arg CB0=$(date +%V) - --build-arg REPO_FILE_URL=$REPO_FILE_URL + --build-arg REPO_FILE_URL=${{ inputs.repo_file_url }} --build-arg UID=$(id -u) - --build-arg FVERSION=${{ env.FVERSION }} - --build-arg JENKINS_URL=${{ env.JENKINS_URL }} + --build-arg FVERSION=${{ steps.build-vars.outputs.fversion }} + --build-arg JENKINS_URL=${{ inputs.jenkins_url }} --tag mock-build utils/rpms - name: Build RPM @@ -109,18 +73,18 @@ runs: docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.distro }} --user build -v "$PWD":"$PWD" -w "$PWD" - -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result + -v "$PWD"/mock_result:/var/lib/mock/${{ steps.build-vars.outputs.chroot_name }}/result --privileged=true - -e DAOS_FULLNAME="$DAOS_FULLNAME" - -e DAOS_EMAIL="$DAOS_EMAIL" - -e DISTRO_VERSION="$DISTRO_VERSION" - -e STAGE_NAME="$STAGE_NAME" - -e CHROOT_NAME="$CHROOT_NAME" - -e ARTIFACTORY_URL="$ARTIFACTORY_URL" - -e REPO_FILE_URL="$REPO_FILE_URL" - -e JENKINS_URL=${{ env.JENKINS_URL }} - -e TARGET="$TARGET" - -e COVFN_DISABLED="$COVFN_DISABLED" + -e DAOS_FULLNAME="${{ inputs.daos_full_name }}" + -e DAOS_EMAIL="${{ inputs.daos_email }}" + -e DISTRO_VERSION="${{ steps.build-vars.outputs.distro_version }}" + -e STAGE_NAME="Build RPM on ${{ steps.build-vars.outputs.build name }}" + -e CHROOT_NAME="${{ steps.build-vars.outputs.chroot_name}}" + -e ARTIFACTORY_URL="${{ inputs.artifactory_url}}" + -e REPO_FILE_URL="${{ inputs.repo_file_url }}" + -e JENKINS_URL=${{ inputs.jenkins_url }} + -e TARGET="${{ inputs.distro }}" + -e COVFN_DISABLED="${{ inputs.covfn_disabled }}" mock-build ci/rpm/build.sh # yamllint enable rule:line-length - name: Build RPM failure log @@ -133,7 +97,7 @@ runs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: ${{ env.STAGE_NAME }} logs + name: ${{ steps.build-vars.outputs.stage_name }} logs path: | mock_result/root.log mock_result/build.log @@ -154,21 +118,23 @@ runs: if: steps.create-repo.outcome == 'success' continue-on-error: true run: . ci/gha_functions.sh; - dnf --disablerepo=\* --repofrompath - testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET - repoquery -a + dnf --disablerepo=\* --repofrompath testrepo,file://${REPO_FILE} repoquery -a + env: + # yamllint disable-line rule:line-length + REPO_FILE: ${{ inputs.repo_path }}${{ github.run_number }}/artifact/artifacts/${{ inputs.distro }} - name: Remove lastSuccessfulBuild link and exit failure if: steps.test-repo.outcome != 'success' - run: rm -f ${REPO_PATH}lastSuccessfulBuild; + run: rm -f ${{ inputs.repo_path }}lastSuccessfulBuild; exit 1 - name: Publish RPMs uses: actions/upload-artifact@v4 with: - name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} RPM repository - path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} + # yamllint disable-line rule:line-length + name: ${{ steps.build-vars.outputs.distro_name }} ${{ steps.build-vars.outputs.distro_version }} RPM repository + path: ${{ inputs.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ inputs.distro }} - name: Update commit status uses: ouzi-dev/commit-status-updater@v2 with: # yamllint disable-line rule:line-length - name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' + name: 'build/Build RPM on ${{ steps.build-vars.outputs.distro_name }} ${{ steps.build-vars.outputs.commit_status_distro_version }}' status: "${{ job.status }}" diff --git a/.github/actions/build-variables/action.yml b/.github/actions/build-variables/action.yml new file mode 100644 index 00000000000..56a28b6b60c --- /dev/null +++ b/.github/actions/build-variables/action.yml @@ -0,0 +1,113 @@ +name: 'Build Variables' +description: 'Set variables for build based upon distro version' + +on: + workflow_call: + inputs: + distro: + type: choice + options: + - el8 + - el9 + - leap15 + version: + type: string + required: true + build_version: + type: string + required: true + +outputs: + fversion: + value: ${{ steps.set-variables.outputs.fversion }} + inst_rpms: + value: ${{ steps.set-variables.outputs.inst_rpms }} + chroot_name: + value: ${{ steps.set-variables.outputs.chroot_name }} + distro_name: + value: ${{ steps.set-variables.outputs.distro_name }} + distro_name_upper: + value: ${{ steps.set-variables.outputs.distro_name_upper }} + distro_name_lower: + value: ${{ steps.set-variables.outputs.distro_name_lower }} + distro_version: + value: ${{ steps.set-variables.outputs.distro_version }} + distro_version_major: + value: ${{ steps.set-variables.outputs.distro_version_major }} + commit_status_distro_version: + value: ${{ steps.set-variables.outputs.commit_status_distro_version }} + openmpi: + value: ${{ steps.set-variables.outputs.openmpi }} + build_chroot: + value: ${{ steps.set-variables.outputs.build_chroot }} + build_name: + value: ${{ steps.set-variables.outputs.build_name }} + distro: + value: ${{ steps.set-variables.outputs.distro }} + provision_distro: + value: ${{ steps.set-variables.outputs.provision_distro }} + local_repo_name: + value: ${{ steps.set-variables.outputs.local_repo_name }} + +runs: + using: composite + steps: + - name: Set variables + id: set-variables + run: | + FVERSION="38" + INST_RPMS="daos-client daos-tests daos-server daos-serialize daos-tests-internal" + case ${{ inputs.distro }} in + 'el8') + CHROOT_NAME="rocky+epel-8-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION="${{ inputs.build_version }}" + DISTRO_VERSION_MAJOR="8" + PROVISION_VERSION_MAJOR="8" + COMMIT_STATUS_DISTRO_VERSION="8" + OPENMPI="openmpi" + ;; + 'el9') + CHROOT_NAME="rocky+epel-9-x86_64" + DISTRO_NAME="EL" + DISTRO_NAME_UPPER="EL" + DISTRO_NAME_LOWER="el" + DISTRO_VERSION="${{ inputs.build_version }}" + DISTRO_VERSION_MAJOR="9" + PROVISION_VERSION_MAJOR="8" + COMMIT_STATUS_DISTRO_VERSION="${{ inputs.build_version }}" + OPENMPI="openmpi" + ;; + 'leap15') + CHROOT_NAME="opensuse-leap-${{ inputs.build_version }}-x86_64" + DISTRO_NAME="Leap" + DISTRO_NAME_UPPER="LEAP" + DISTRO_NAME_LOWER="leap" + DISTRO_VERSION="${{ inputs.build_version }}" + DISTRO_VERSION_MAJOR="15" + PROVISION_VERSION_MAJOR="15" + COMMIT_STATUS_DISTRO_VERSION="${{ inputs.build_version }}" + OPENMPI="openmpi3" + ;; + esac + echo "fversion=${FVERSION}" >> $GITHUB_OUTPUT + echo "inst_rpms=${INST_RPMS}" >> $GITHUB_OUTPUT + echo "chroot_name=${CHROOT_NAME}" >> $GITHUB_OUTPUT + echo "distro_name=${DISTRO_NAME}" >> $GITHUB_OUTPUT + echo "distro_name_upper=${DISTRO_NAME_UPPER}" >> $GITHUB_OUTPUT + echo "distro_name_lower=${DISTRO_NAME_LOWER}" >> $GITHUB_OUTPUT + echo "distro_version=${DISTRO_VERSION}" >> $GITHUB_OUTPUT + echo "distro_version_major=${DISTRO_VERSION_MAJOR}" >> $GITHUB_OUTPUT + echo "commit_status_distro_version=${COMMIT_STATUS_DISTRO_VERSION}" >> $GITHUB_OUTPUT + echo "openmpi=${OPENMPI}" >> $GITHUB_OUTPUT + echo "build_chroot=/var/lib/mock/${CHROOT_NAME}-${{ github.run_id }}/" >> $GITHUB_OUTPUT + echo "build_name=${DISTRO_NAME} ${DISTRO_VERSION}" >> $GITHUB_OUTPUT + echo "distro=${DISTRO_NAME_UPPER}_${DISTRO_VERSION_MAJOR}" >> $GITHUB_OUTPUT + echo "provision_distro=${DISTRO_NAME_UPPER}_${PROVISION_VERSION_MAJOR}" >> $GITHUB_OUTPUT + echo "local_repo_name=${DISTRO_NAME_UPPER}_${PROVISION_VERSION_MAJOR}" >> $GITHUB_OUTPUT + + # echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV + # echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ + # $GITHUB_ENV diff --git a/.github/actions/functional-test/action.yml b/.github/actions/functional-test/action.yml index 9a4ca089f23..4d3172bd8dd 100644 --- a/.github/actions/functional-test/action.yml +++ b/.github/actions/functional-test/action.yml @@ -10,114 +10,55 @@ on: - el8 - el9 - leap15 + version: + type: string + required: true + build_version: + type: string + required: true + cluster_label: + type: string + default: true tags: type: string required: true ftest_args: type: string - default: --nvme=auto - -env: - CONFIG_POWER_ONLY: false - PRAGMA_SUFFIX: -vm - OPERATIONS_EMAIL: phil.henderson@intel.com - TEST_RPMS: true - COMMIT_MESSAGE: - JENKINS_URL: https://build.hpdd.intel.com/ - REPOSITORY_URL: https://repo.dc.hpdd.intel.com/ - REMOVE_EXISTING_RPMS: false - # TODO -- this should be on stable, backedup storage - ARTIFACTS_URL: file:///scratch/job_repos/ - REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ - # keep VS Code's GHA linting happy - NODESTRING: - CP_PR_REPOS: - CP_FEATURES: - CP_TEST_TAG: - CP_EL8_VM9_LABEL: - CP_EL9_VM9_LABEL: - CP_LEAP15_VM9_LABEL: - CP_PRIORITY: - CP_EL8_VERSION: - CP_EL9_VERSION: - CP_LEAP15_VERSION: - DISTRO: - CLUSTER_REQUEST_reqid: - STAGE_NAME: - QUEUE_URL: - LABEL: - DISTRO_NAME: - DISTRO_VERSION: - COMMIT_STATUS_DISTRO_VERSION: - FTEST_ARG: + default: "" + jenkins_url: + type: string + default: https://build.hpdd.intel.com/ runs: using: "composite" steps: + - name: Set build variables + id: build-vars + uses: ./.github/actions/build-variables + with: + distro: ${{ inputs.distro }} + version: ${{ inputs.version }} + build_version: ${{ inputs.build_version }} - name: Set variables run: | set -eux env - STAGE_TAGS="vm" - FTEST_ARG={{ $inputs.ftest_args }} - INST_RPMS="daos-client daos-tests daos-server daos-serialize daos-tests-internal" - case "${{ inputs.distro }}" in - 'el8') - CHROOT_NAME="rocky+epel-8-x86_64" - DISTRO_NAME="EL" - DISTRO_NAME_UPPER="EL" - DISTRO_NAME_LOWER="el" - DISTRO_VERSION="${{ env.CP_EL8_VERSION && - env.CP_EL8_VERSION || env.EL8_VERSION }}" - DISTRO_VERSION_MAJOR="8" - OPENMPI="openmpi" - LABEL="${{ env.CP_EL8_VM9_LABEL && - env.CP_EL8_VM9_LABEL || 'ci_vm9' }}" - ;; - 'el9') - CHROOT_NAME="rocky+epel-9-x86_64" - DISTRO_NAME="EL" - DISTRO_NAME_UPPER="EL" - DISTRO_NAME_LOWER="el" - DISTRO_VERSION="${{ env.CP_EL9_VERSION && - env.CP_EL9_VERSION || env.EL9_VERSION }}" - DISTRO_VERSION_MAJOR="9" - PROV_DISTRO_VERSION_MAJOR="8" - OPENMPI="openmpi" - LABEL="${{ env.CP_EL9_VM9_LABEL && - env.CP_EL9_VM9_LABEL || 'ci_vm9' }}" - ;; - 'leap15') - CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || - env.LEAP15_VERSION }}-x86_64" - DISTRO_NAME="Leap" - DISTRO_NAME_UPPER="LEAP" - DISTRO_NAME_LOWER="leap" - DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && - env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" - DISTRO_VERSION_MAJOR="15" - OPENMPI="openmpi3" - LABEL="${{ env.CP_LEAP15_VM9_LABEL && - env.CP_LEAP15_VM9_LABEL || 'ci_vm9' }}" - ;; - esac - echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV - echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV - echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV - echo "DISTRO_WITH_VERSION=$DISTRO_NAME_LOWER$DISTRO_VERSION" >> $GITHUB_ENV - echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV - echo "STAGE_NAME=Functional on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV - echo "STAGE_TAGS=$STAGE_TAGS" >> $GITHUB_ENV - echo "FTEST_ARG=$FTEST_ARG" >> $GITHUB_ENV - echo "DISTRO=${DISTRO_NAME_UPPER}_$DISTRO_VERSION_MAJOR" >> $GITHUB_ENV - echo -n "PROVISION_DISTRO=${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV - echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}" >> $GITHUB_ENV - echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV - echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ - $GITHUB_ENV - echo "LABEL=$LABEL" >> $GITHUB_ENV - echo "INST_RPMS=$INST_RPMS" >> $GITHUB_ENV + echo "CHROOT_NAME=${{ steps.build-vars.outputs.chroot_name }}" >> $GITHUB_ENV + echo "DISTRO_NAME=${{ steps.build-vars.outputs.distro_name }}" >> $GITHUB_ENV + echo "DISTRO_VERSION=${{ steps.build-vars.outputs.distro_version }}" >> $GITHUB_ENV + echo -n "DISTRO_WITH_VERSION=" >> $GITHUB_ENV + echo -n "${{ steps.build-vars.outputs.distro_name_lower }}" >> $GITHUB_ENV + echo "${{ steps.build-vars.outputs.distro_version }}" >> $GITHUB_ENV + echo "BUILD_CHROOT=${{ steps.build-vars.outputs.build_chroot }}" >> $GITHUB_ENV + echo "STAGE_NAME=Functional on ${{ steps.build-vars.outputs.build_name }}" >> $GITHUB_ENV + echo "STAGE_TAGS=${{ inputs.tags }}" >> $GITHUB_ENV + echo "FTEST_ARG=${{ inputs.ftest_args }}" >> $GITHUB_ENV + echo "DISTRO=${{ steps.build-vars.outputs.distro }}" >> $GITHUB_ENV + echo "PROVISION_DISTRO=${{ steps.build-vars.outputs.provision_distro }}" >> $GITHUB_ENV + echo -n "DAOS_STACK_${{ steps.build-vars.outputs.local_repo_name }}" >> $GITHUB_ENV + echo "_LOCAL_REPO=not_used" >> $GITHUB_ENV + echo "LABEL=${{ inputs.cluster_label }}" >> $GITHUB_ENV + echo "INST_RPMS=${{ steps.build-vars.outputs.inst_rpms }}" >> $GITHUB_ENV - name: Checkout code uses: actions/checkout@v4 with: @@ -127,27 +68,23 @@ runs: - name: Request and Provision a Cluster timeout-minutes: 7200 uses: ./.github/actions/provision-cluster - with: - condition: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && \ - env.CP_SKIP_FUNC_TEST != 'true' - name: Run Test timeout-minutes: 7200 - if: env.CP_SKIP_FUNC_TEST-${{ env.DISTRO }} != 'true' && env.CP_SKIP_FUNC_TEST != 'true' id: run-test run: | . ci/gha_functions.sh NODE_COUNT="$NODE_COUNT" - TEST_TAG={{ $inputs.tags }} - FTEST_ARG="${{ env.FTEST_ARG }}" ci/functional/test_main.sh + TEST_TAG=${{ inputs.tags }} + FTEST_ARG="${{ inputs.ftest_args }}" ci/functional/test_main.sh - name: Cancel cluster request (if cancelled after requesting) if: cancelled() run: | set -eux . ci/gha_functions.sh - if ! JENKINS_URL="${{ env.JENKINS_URL }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ + if ! JENKINS_URL="${{ inputs.jenkins_url }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ cancel_provision; then # probably already provisioned and needs un-provisioning - if ! cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}"; then + if ! cleanup_provision_request "${{ inputs.cluster_request_reqid }}"; then exit 1 fi fi diff --git a/.github/workflows/code-coverage-test.yml b/.github/workflows/code-coverage-test.yml index 7f80153f78e..1d13c25c290 100644 --- a/.github/workflows/code-coverage-test.yml +++ b/.github/workflows/code-coverage-test.yml @@ -1,5 +1,30 @@ name: Code Coverage Test +env: + ARTIFACTORY_URL: https://artifactory.dc.hpdd.intel.com/ + JENKINS_URL: https://build.hpdd.intel.com/ + REPO_FILE_URL: https://artifactory.dc.hpdd.intel.com/artifactory/repo-files/ + DAOS_EMAIL: phil.henderson@intel.com + DAOS_FULLNAME: daos-stack + DISTRO_REPOS: disabled + DOCKER_BUILDKIT: 0 + ARTIFACTS_URL: file:///scratch/job_repos/ + MOCK_OPTIONS: --uniqueext=${{ github.run_id }} + PR_NUM: ${{ github.event.pull_request.number }} + + # TODO -- this should be on stable, backed up storage, not /scratch + # yamllint disable-line rule:line-length + REPO_PATH: /scratch/job_repos/daos-stack/job/daos/job/PR-${{ github.event.pull_request.number }}/ + + # TODO: we really need to define a list of supported versions (ideally it's no more than 2) + # build is done on the lowest version and test on the highest with a "sanity test" + # stage done on all versions in the list except the highest + EL8_BUILD_VERSION: 8.6 + EL8_VERSION: 8.8 + EL9_BUILD_VERSION: 9 + EL9_VERSION: 9 + LEAP15_VERSION: 15.5 + on: pull_request: # Remove after testing schedule: @@ -28,7 +53,15 @@ jobs: uses: ./.github/actions/build-rpm with: distro: el8 + version: ${{ env.EL8_VERSION }} + build_version: ${{ env.EL8_BUILD_VERSION }} covfn_disabled: false + repo_file_url: ${{ env.REPO_FILE_URL }} + jenkins_url: ${{ env.JENKINS_URL }} + daos_full_name: ${{ env.DAOS_FULLNAME }} + daos_email: ${{ env.DAOS_EMAIL }} + artifactory_url: ${{ env.ARTIFACTORY_URL }} + repo_path: ${{ env.REPO_PATH }} functional-test: name: Bullseye Functional Test @@ -51,4 +84,7 @@ jobs: uses: ./.github/actions/functional-test with: distro: el8 + version: ${{ env.EL8_VERSION }} + build_version: ${{ env.EL8_BUILD_VERSION }} + cluster_label: ci_vm9 tags: test_one_pool_vm From 3520c1a135350069ad41450d9af4406605bdf9ed Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 17 Oct 2024 14:41:00 -0400 Subject: [PATCH 08/16] Add missing 'shell: bash'. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/build-rpm/action.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-rpm/action.yml b/.github/actions/build-rpm/action.yml index 7312e0b5670..8503a8b62c4 100644 --- a/.github/actions/build-rpm/action.yml +++ b/.github/actions/build-rpm/action.yml @@ -64,6 +64,7 @@ runs: --build-arg JENKINS_URL=${{ inputs.jenkins_url }} --tag mock-build utils/rpms + shell: bash - name: Build RPM id: build-rpm continue-on-error: true @@ -78,7 +79,7 @@ runs: -e DAOS_FULLNAME="${{ inputs.daos_full_name }}" -e DAOS_EMAIL="${{ inputs.daos_email }}" -e DISTRO_VERSION="${{ steps.build-vars.outputs.distro_version }}" - -e STAGE_NAME="Build RPM on ${{ steps.build-vars.outputs.build name }}" + -e STAGE_NAME="Build RPM on ${{ steps.build-vars.outputs.build_name }}" -e CHROOT_NAME="${{ steps.build-vars.outputs.chroot_name}}" -e ARTIFACTORY_URL="${{ inputs.artifactory_url}}" -e REPO_FILE_URL="${{ inputs.repo_file_url }}" @@ -86,7 +87,7 @@ runs: -e TARGET="${{ inputs.distro }}" -e COVFN_DISABLED="${{ inputs.covfn_disabled }}" mock-build ci/rpm/build.sh - # yamllint enable rule:line-length + shell: bash - name: Build RPM failure log id: build-rpm-fail-log continue-on-error: true @@ -108,17 +109,20 @@ runs: rm -f ${REPO_PATH}last{,Successful}Build; ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild; ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild + shell: bash - name: Create repo id: create-repo if: steps.build-rpm.outcome == 'success' continue-on-error: true run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh + shell: bash - name: Test repo id: test-repo if: steps.create-repo.outcome == 'success' continue-on-error: true run: . ci/gha_functions.sh; dnf --disablerepo=\* --repofrompath testrepo,file://${REPO_FILE} repoquery -a + shell: bash env: # yamllint disable-line rule:line-length REPO_FILE: ${{ inputs.repo_path }}${{ github.run_number }}/artifact/artifacts/${{ inputs.distro }} @@ -126,6 +130,7 @@ runs: if: steps.test-repo.outcome != 'success' run: rm -f ${{ inputs.repo_path }}lastSuccessfulBuild; exit 1 + shell: bash - name: Publish RPMs uses: actions/upload-artifact@v4 with: From 5c5ee7d3fed94d6e997dc7ad484b6e3ec4558e66 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 17 Oct 2024 14:43:23 -0400 Subject: [PATCH 09/16] Fix missing required property. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/build-rpm/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/build-rpm/action.yml b/.github/actions/build-rpm/action.yml index 8503a8b62c4..5769808e16b 100644 --- a/.github/actions/build-rpm/action.yml +++ b/.github/actions/build-rpm/action.yml @@ -94,6 +94,7 @@ runs: if: steps.build-rpm.outcome != 'success' run: cat mock_result/root.log; cat mock_result/build.log + shell: bash - name: Save RPM build logs continue-on-error: true uses: actions/upload-artifact@v4 From ccc327b69a7c8979775c6d8d4c609bba7f47b132 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 17 Oct 2024 14:46:50 -0400 Subject: [PATCH 10/16] Fix build-variables. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/build-variables/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/build-variables/action.yml b/.github/actions/build-variables/action.yml index 56a28b6b60c..3596ce5c52b 100644 --- a/.github/actions/build-variables/action.yml +++ b/.github/actions/build-variables/action.yml @@ -111,3 +111,4 @@ runs: # echo -n "DAOS_STACK_${DISTRO_NAME_UPPER}_" >> $GITHUB_ENV # echo "${PROV_DISTRO_VERSION_MAJOR:-$DISTRO_VERSION_MAJOR}_LOCAL_REPO=not_used" >> \ # $GITHUB_ENV + shell: bash From caf8861cb4411a1eef9c193eba158c67e69ee41d Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 17 Oct 2024 16:14:04 -0400 Subject: [PATCH 11/16] More fixes. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/functional-test/action.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/functional-test/action.yml b/.github/actions/functional-test/action.yml index 4d3172bd8dd..d2432868ba0 100644 --- a/.github/actions/functional-test/action.yml +++ b/.github/actions/functional-test/action.yml @@ -59,6 +59,7 @@ runs: echo "_LOCAL_REPO=not_used" >> $GITHUB_ENV echo "LABEL=${{ inputs.cluster_label }}" >> $GITHUB_ENV echo "INST_RPMS=${{ steps.build-vars.outputs.inst_rpms }}" >> $GITHUB_ENV + shell: bash - name: Checkout code uses: actions/checkout@v4 with: @@ -68,6 +69,7 @@ runs: - name: Request and Provision a Cluster timeout-minutes: 7200 uses: ./.github/actions/provision-cluster + shell: bash - name: Run Test timeout-minutes: 7200 id: run-test @@ -76,6 +78,7 @@ runs: NODE_COUNT="$NODE_COUNT" TEST_TAG=${{ inputs.tags }} FTEST_ARG="${{ inputs.ftest_args }}" ci/functional/test_main.sh + shell: bash - name: Cancel cluster request (if cancelled after requesting) if: cancelled() run: | @@ -88,6 +91,7 @@ runs: exit 1 fi fi + shell: bash - name: Job cleanup if: (!cancelled() && (success() || failure())) run: | @@ -95,6 +99,7 @@ runs: . ci/gha_functions.sh NODELIST=${{ env.NODESTRING }} ci/functional/job_cleanup.sh || true cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}" + shell: bash - name: Publish test results if: (!cancelled()) && (success() || failure()) && steps.run-test.outcome != 'skipped' From cb2c7ab3acd253635bc16d817c54938f1ca266a5 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 17 Oct 2024 16:21:23 -0400 Subject: [PATCH 12/16] Updates. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/build-rpm/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-rpm/action.yml b/.github/actions/build-rpm/action.yml index 5769808e16b..f7b827ca2a2 100644 --- a/.github/actions/build-rpm/action.yml +++ b/.github/actions/build-rpm/action.yml @@ -115,8 +115,10 @@ runs: id: create-repo if: steps.build-rpm.outcome == 'success' continue-on-error: true - run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh + run: ci/rpm/create_repo.sh shell: bash + env: + TARGET: ${{ inputs.distro }} - name: Test repo id: test-repo if: steps.create-repo.outcome == 'success' From 8a76f64392995257cfa5fc0819c5bd433462cad0 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 18 Oct 2024 12:40:48 -0400 Subject: [PATCH 13/16] functional-test updates. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/functional-test/action.yml | 8 ++++---- .github/workflows/code-coverage-test.yml | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/actions/functional-test/action.yml b/.github/actions/functional-test/action.yml index d2432868ba0..2dc7a8df9c9 100644 --- a/.github/actions/functional-test/action.yml +++ b/.github/actions/functional-test/action.yml @@ -24,10 +24,13 @@ on: required: true ftest_args: type: string - default: "" + default: "--nvme=auto" jenkins_url: type: string default: https://build.hpdd.intel.com/ + secrets: + GITHUB_TOKEN: + required: true runs: using: "composite" @@ -67,11 +70,8 @@ runs: fetch-depth: 500 ref: ${{ github.event.pull_request.head.sha }} - name: Request and Provision a Cluster - timeout-minutes: 7200 uses: ./.github/actions/provision-cluster - shell: bash - name: Run Test - timeout-minutes: 7200 id: run-test run: | . ci/gha_functions.sh diff --git a/.github/workflows/code-coverage-test.yml b/.github/workflows/code-coverage-test.yml index 1d13c25c290..00cb3a8eff0 100644 --- a/.github/workflows/code-coverage-test.yml +++ b/.github/workflows/code-coverage-test.yml @@ -80,7 +80,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4.1.1 - - name: Build RPMs + - name: Run function test + timeout-minutes: 2880 uses: ./.github/actions/functional-test with: distro: el8 @@ -88,3 +89,6 @@ jobs: build_version: ${{ env.EL8_BUILD_VERSION }} cluster_label: ci_vm9 tags: test_one_pool_vm + jenkins_url: ${{ env.JENKINS_URL }} + secrets: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From c782bbcf7ea0396af9953ffcc15e75fc5a7b69a1 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 18 Oct 2024 12:51:01 -0400 Subject: [PATCH 14/16] Fix secrets error. Required-githooks: true Signed-off-by: Phil Henderson --- .github/workflows/code-coverage-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/code-coverage-test.yml b/.github/workflows/code-coverage-test.yml index 00cb3a8eff0..917a91f414d 100644 --- a/.github/workflows/code-coverage-test.yml +++ b/.github/workflows/code-coverage-test.yml @@ -90,5 +90,4 @@ jobs: cluster_label: ci_vm9 tags: test_one_pool_vm jenkins_url: ${{ env.JENKINS_URL }} - secrets: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From 1d6949b77ad95de9406f1ccab6674607ef5f3c3b Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 18 Oct 2024 14:55:05 -0400 Subject: [PATCH 15/16] Pass secret as input to composite action. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/functional-test/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/functional-test/action.yml b/.github/actions/functional-test/action.yml index 2dc7a8df9c9..d36e1ffd5ea 100644 --- a/.github/actions/functional-test/action.yml +++ b/.github/actions/functional-test/action.yml @@ -28,7 +28,6 @@ on: jenkins_url: type: string default: https://build.hpdd.intel.com/ - secrets: GITHUB_TOKEN: required: true @@ -106,7 +105,7 @@ runs: uses: EnricoMi/publish-unit-test-result-action@v2 with: check_name: ${{ env.STAGE_NAME }} Test Results (old) - github_token: ${{ secrets.GITHUB_TOKEN }} + github_token: ${{ inputs.GITHUB_TOKEN }} junit_files: ${{ env.STAGE_NAME }}/**/results.xml - name: Publish artifacts if: (!cancelled()) && (success() || failure()) && From cc7116aa50196add9105366a225333715ef24f78 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 18 Oct 2024 15:36:40 -0400 Subject: [PATCH 16/16] Fix typo. Required-githooks: true Signed-off-by: Phil Henderson --- .github/actions/functional-test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/functional-test/action.yml b/.github/actions/functional-test/action.yml index d36e1ffd5ea..aa145bba844 100644 --- a/.github/actions/functional-test/action.yml +++ b/.github/actions/functional-test/action.yml @@ -86,7 +86,7 @@ runs: if ! JENKINS_URL="${{ inputs.jenkins_url }}" QUEUE_URL="${{ env.QUEUE_URL }}" \ cancel_provision; then # probably already provisioned and needs un-provisioning - if ! cleanup_provision_request "${{ inputs.cluster_request_reqid }}"; then + if ! cleanup_provision_request "${{ env.CLUSTER_REQUEST_reqid }}"; then exit 1 fi fi