diff --git a/.github/workflows/build-test-release.yaml b/.github/workflows/build-test-release.yaml index ddc75c4bd..009e47d29 100644 --- a/.github/workflows/build-test-release.yaml +++ b/.github/workflows/build-test-release.yaml @@ -16,7 +16,15 @@ permissions: statuses: write jobs: + test-skip: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + compliance-copyrights: + needs: + - test-skip + if: false runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -24,7 +32,11 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + pre-commit: + needs: + - compliance-copyrights + if: ${{ success() || failure() }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -38,7 +50,8 @@ jobs: publish: needs: - compliance-copyrights - - pre-commit +# if: ${{ always() }} + if: ${{always() && needs.compliance-copyrights.result != 'skipped'}} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/reusable-build-test-release.yml b/.github/workflows/reusable-build-test-release.yml index 2966fb17a..61463f8b9 100644 --- a/.github/workflows/reusable-build-test-release.yml +++ b/.github/workflows/reusable-build-test-release.yml @@ -88,7 +88,7 @@ on: permissions: contents: read packages: read -concurrency: +concurrency: group: ${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: @@ -240,16 +240,45 @@ jobs: persist-credentials: false - id: matrix uses: splunk/addonfactory-test-matrix-action@v2.1 + - name: job summary + run: | + splunk_version_list=$(echo '${{ steps.matrix.outputs.supportedSplunk }}' | jq -r '.[].version') + sc4s_version_list=$(echo '${{ steps.matrix.outputs.supportedSC4S }}' | jq -r '.[].version') + echo "## Summary of Versions Used" >> $GITHUB_STEP_SUMMARY + echo "- **Splunk versions used:** (${splunk_version_list})" >> $GITHUB_STEP_SUMMARY + echo "- **SC4S versions used:** (${sc4s_version_list})" >> $GITHUB_STEP_SUMMARY + echo "- Browser: Chrome" >> $GITHUB_STEP_SUMMARY fossa-scan: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: run fossa analyze and create report + id: fossa-scan run: | + # Install the latest version of FOSSA CLI curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash - fossa analyze --debug + + # Run FOSSA analyze, display the output in real-time, and capture it for further processing + fossa_analyze_output=$(fossa analyze --debug 2>&1 | tee /tmp/fossa_analyze_output.txt) + exit_code=$? + + # Extract the FOSSA report URL from the captured output + # Using '|| true' ensures that the script doesn't fail if grep returns no matches + FOSSA_REPORT_URL=$(grep -o 'https://app.fossa.com[^ ]*' /tmp/fossa_analyze_output.txt || true) + + # Print the extracted URL (if any) + echo "url=$FOSSA_REPORT_URL" + + # Save the FOSSA report URL to the GitHub Actions output + echo "FOSSA_REPORT_URL=$FOSSA_REPORT_URL" >> "$GITHUB_OUTPUT" + + # Run the FOSSA report command, capturing the output to a file fossa report attribution --format text --timeout 600 > /tmp/THIRDPARTY + + # Exit with the same exit code as the FOSSA analyze command + exit $exit_code + env: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} - name: upload THIRDPARTY file @@ -257,6 +286,10 @@ jobs: with: name: THIRDPARTY path: /tmp/THIRDPARTY + - name: job summary + if: success() || failure() + run: | + echo "FOSSA Report: ${{ steps.fossa-scan.outputs.FOSSA_REPORT_URL }}" >> $GITHUB_STEP_SUMMARY fossa-test: continue-on-error: true @@ -312,7 +345,7 @@ jobs: with: extra_args: -x .github/workflows/exclude-patterns.txt --json --only-verified version: 3.77.0 - + semgrep: runs-on: ubuntu-latest name: security-sast-semgrep @@ -352,7 +385,7 @@ jobs: ucc_modinput_tests="false" fi echo "ucc_modinput_tests=$ucc_modinput_tests" >> "$GITHUB_OUTPUT" - + run-unit-tests: name: test-unit-python3-${{ matrix.python-version }} if: ${{ needs.test-inventory.outputs.unit == 'true' }} @@ -364,7 +397,7 @@ jobs: matrix: python-version: - "3.7" - permissions: + permissions: actions: read deployments: read contents: read @@ -398,6 +431,19 @@ jobs: run: cp tests/unit/pytest-ci.ini pytest.ini - name: Run Pytest with coverage run: poetry run pytest --cov=./ --cov-report=xml --junitxml=test-results/junit.xml tests/unit + - name : Job summary + run: | + sudo apt-get install -y libxml2-utils + junit_xml_path="test-results/junit.xml" + total_tests=$(xmllint --xpath 'sum(//testsuite/@tests)' $junit_xml_path) + failures=$(xmllint --xpath 'sum(//testsuite/@failures)' $junit_xml_path) + errors=$(xmllint --xpath 'sum(//testsuite/@errors)' $junit_xml_path) + skipped=$(xmllint --xpath 'sum(//testsuite/@skipped)' $junit_xml_path) + passed=$((total_tests - failures - errors - skipped)) + + echo "| Total Tests | Passed | Failed | Errors | Skipped |" >> $GITHUB_STEP_SUMMARY + echo "| ----------- | ------------ | ------------ | ------------- | ------------- |" >> $GITHUB_STEP_SUMMARY + echo "| $total_tests | $passed | $failures | $errors | $skipped |" >> $GITHUB_STEP_SUMMARY - uses: actions/upload-artifact@v4 if: success() || failure() with: @@ -415,7 +461,7 @@ jobs: matrix: python-version: - "3.9" - permissions: + permissions: actions: read deployments: read contents: read @@ -449,6 +495,20 @@ jobs: run: cp tests/unit/pytest-ci.ini pytest.ini - name: Run Pytest with coverage run: poetry run pytest --cov=./ --cov-report=xml --junitxml=test-results/junit.xml tests/unit + - name : Job summary + run: | + sudo apt-get install -y libxml2-utils + junit_xml_path="test-results/junit.xml" + total_tests=$(xmllint --xpath 'sum(//testsuite/@tests)' $junit_xml_path) + failures=$(xmllint --xpath 'sum(//testsuite/@failures)' $junit_xml_path) + errors=$(xmllint --xpath 'sum(//testsuite/@errors)' $junit_xml_path) + skipped=$(xmllint --xpath 'sum(//testsuite/@skipped)' $junit_xml_path) + passed=$((total_tests - failures - errors - skipped)) + echo "| Total Tests | Passed Tests | Failed Tests | Errored Tests | Skipped Tests |" >> $GITHUB_STEP_SUMMARY + echo "| ----------- | ------------ | ------------ | ------------- | ------------- |" >> $GITHUB_STEP_SUMMARY + echo "| $total_tests | $passed | $failures | $errors | $skipped |" >> $GITHUB_STEP_SUMMARY + + - uses: actions/upload-artifact@v4 if: success() || failure() with: @@ -757,7 +817,7 @@ jobs: - build - test-inventory if: ${{ !cancelled() && needs.build.result == 'success' && needs.test-inventory.outputs.requirement_test == 'true' }} - permissions: + permissions: actions: read deployments: read contents: read @@ -970,7 +1030,6 @@ jobs: matrix: splunk: ${{ fromJson(needs.meta.outputs.matrix_supportedSplunk) }} sc4s: ${{ fromJson(needs.meta.outputs.matrix_supportedSC4S) }} - container: image: ghcr.io/splunk/workflow-engine-base:4.1.0 env: @@ -982,7 +1041,7 @@ jobs: SPLUNK_VERSION_BASE: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} TEST_TYPE: "knowledge" TEST_ARGS: "" - permissions: + permissions: actions: read deployments: read contents: read @@ -1176,6 +1235,23 @@ jobs: name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + - name: Parse JUnit XML + if: ${{ !cancelled() }} + run: | + apt-get install -y libxml2-utils + junit_xml_path="${{ needs.setup.outputs.directory-path }}/test-results/*.xml" + total_tests=$(xmllint --xpath 'sum(//testsuite/@tests)' $junit_xml_path) + failures=$(xmllint --xpath 'sum(//testsuite/@failures)' $junit_xml_path) + errors=$(xmllint --xpath 'sum(//testsuite/@errors)' $junit_xml_path) + skipped=$(xmllint --xpath 'sum(//testsuite/@skipped)' $junit_xml_path) + passed=$((total_tests - failures - errors - skipped)) + echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} |$total_tests |$passed |$failures |$errors | $skipped |${{steps.test_report.outputs.url_html}}" > summary.txt + - name: Upload-${{ matrix.splunk.version }}-${{ matrix.sc4s.version }}-${{ env.TEST_TYPE }} + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: summary-ko-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} + path: summary.txt - name: pull diag from s3 bucket if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} run: | @@ -1188,6 +1264,22 @@ jobs: path: | ${{ needs.setup.outputs.directory-path }}/diag* + knowledge-tests-report: + needs: run-knowledge-tests + runs-on: ubuntu-latest + if: success() || failure() + steps: + - name: Download all summaries + uses: actions/download-artifact@v4 + with: + pattern: summary-ko* + - name: Combine summaries into a table + run: | + echo "| Job | Total Tests | Passed | Failed | Errors| Skipped | Report Link" >> $GITHUB_STEP_SUMMARY + echo "| ---------- | ----------- | ------ | ------ | ------ | ------- | ------ |" >> $GITHUB_STEP_SUMMARY + for file in summary-ko*/summary.txt; do + cat "$file" >> $GITHUB_STEP_SUMMARY + done run-requirement-tests: if: ${{ !cancelled() && needs.build.result == 'success' && needs.test-inventory.outputs.requirement_test == 'true' && needs.setup-workflow.outputs.execute-requirement-labeled == 'true' }} @@ -1214,7 +1306,7 @@ jobs: ARGO_NAMESPACE: ${{ needs.setup.outputs.argo-namespace }} TEST_TYPE: "requirement_test" TEST_ARGS: "" - permissions: + permissions: actions: read deployments: read contents: read @@ -1388,6 +1480,24 @@ jobs: name: splunk ${{ matrix.splunk.version }} ${{ env.TEST_TYPE }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + - name: Parse JUnit XML + if: ${{ !cancelled() }} + run: | + apt-get install -y libxml2-utils + junit_xml_path="${{ needs.setup.outputs.directory-path }}/test-results/*.xml" + total_tests=$(xmllint --xpath 'sum(//testsuite/@tests)' $junit_xml_path) + failures=$(xmllint --xpath 'sum(//testsuite/@failures)' $junit_xml_path) + errors=$(xmllint --xpath 'sum(//testsuite/@errors)' $junit_xml_path) + skipped=$(xmllint --xpath 'sum(//testsuite/@skipped)' $junit_xml_path) + passed=$((total_tests - failures - errors - skipped)) + echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} |$total_tests |$passed |$failures |$errors |$skipped |${{steps.test_report.outputs.url_html}}" > summary.txt + - name: Upload-${{ matrix.splunk.version }}-${{ matrix.sc4s.version }}-${{ env.TEST_TYPE }} + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: summary-scripted-input-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} + path: summary.txt + - name: pull diag from s3 bucket if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} run: | @@ -1400,6 +1510,23 @@ jobs: path: | ${{ needs.setup.outputs.directory-path }}/diag* + Requirement-input-tests-report: + needs: run-requirement-tests + runs-on: ubuntu-latest + if: success() || failure() + steps: + - name: Download all summaries + uses: actions/download-artifact@v4 + with: + pattern: summary-requirement* + - name: Combine summaries into a table + run: | + echo "| Job | Total Tests | Passed | Failed | Errors| Skipped | Report Link" >> $GITHUB_STEP_SUMMARY + echo "| ---------- | ----------- | ------ | ------ | ------ | ------- | ------ |" >> $GITHUB_STEP_SUMMARY + for file in summary-requirement*/summary.txt; do + cat "$file" >> $GITHUB_STEP_SUMMARY + done + run-ui-tests: if: ${{ !cancelled() && needs.build.result == 'success' && needs.test-inventory.outputs.ui == 'true' && needs.setup-workflow.outputs.execute-ui-labeled == 'true' }} needs: @@ -1621,18 +1748,52 @@ jobs: name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.browser }} ${{ matrix.vendor-version.image }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + - name: Parse JUnit XML + if: ${{ !cancelled() }} + run: | + apt-get install -y libxml2-utils + junit_xml_path="${{ needs.setup.outputs.directory-path }}/test-results/*.xml" + total_tests=$(xmllint --xpath 'sum(//testsuite/@tests)' $junit_xml_path) + failures=$(xmllint --xpath 'sum(//testsuite/@failures)' $junit_xml_path) + errors=$(xmllint --xpath 'sum(//testsuite/@errors)' $junit_xml_path) + skipped=$(xmllint --xpath 'sum(//testsuite/@skipped)' $junit_xml_path) + passed=$((total_tests - failures - errors - skipped)) + echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.browser }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} |$total_tests |$passed |$failures |$errors |$skipped |${{steps.test_report.outputs.url_html}}" > summary.txt + - name: Upload-artifact-for-github-summary + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: summary-ui-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ matrix.browser }}-${{ matrix.vendor-version.image }}-${{ matrix.marker }}-artifact + path: summary.txt - name: pull diag from s3 bucket if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} run: | echo "pulling diag" aws s3 cp s3://${{ needs.setup.outputs.s3-bucket }}/diag-${{ steps.create-job-name.outputs.job-name }}/diag-${{ steps.create-job-name.outputs.job-name }}.tgz ${{ needs.setup.outputs.directory-path }}/ - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v3 if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} with: name: archive splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.browser }} ${{ matrix.vendor-version.image }} tests diag path: | ${{ needs.setup.outputs.directory-path }}/diag* + UI-tests-report: + needs: run-ui-tests + runs-on: ubuntu-latest + if: success() || failure() + steps: + - name: Download all summaries + uses: actions/download-artifact@v4 + with: + pattern: summary-ui* + - name: Combine summaries into a table + run: | + echo "| Job | Total Tests | Passed | Failed | Errors| Skipped | Report Link" >> $GITHUB_STEP_SUMMARY + echo "| ---------- | ----------- | ------ | ------ | ------ | ------- | ------ |" >> $GITHUB_STEP_SUMMARY + for file in summary-ui-*/summary.txt; do + cat "$file" >> $GITHUB_STEP_SUMMARY + done + run-modinput-tests: if: ${{ !cancelled() && needs.build.result == 'success' && needs.test-inventory.outputs.modinput_functional == 'true' && needs.setup-workflow.outputs.execute-modinput-labeled == 'true' }} needs: @@ -1852,6 +2013,23 @@ jobs: name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + - name: Parse JUnit XML + if: ${{ !cancelled() }} + run: | + apt-get install -y libxml2-utils + junit_xml_path="${{ needs.setup.outputs.directory-path }}/test-results/*.xml" + total_tests=$(xmllint --xpath 'sum(//testsuite/@tests)' $junit_xml_path) + failures=$(xmllint --xpath 'sum(//testsuite/@failures)' $junit_xml_path) + errors=$(xmllint --xpath 'sum(//testsuite/@errors)' $junit_xml_path) + skipped=$(xmllint --xpath 'sum(//testsuite/@skipped)' $junit_xml_path) + passed=$((total_tests - failures - errors - skipped)) + echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.marker }} ${{ matrix.vendor-version.image }} |$total_tests |$passed |$failures |$errors | $skipped |${{steps.test_report.outputs.url_html}}" > summary.txt + - name: Upload-artifact-for-github-summary + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: summary-modinput-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }}-${{ matrix.vendor-version.image }}-${{ matrix.marker }}-artifact + path: summary.txt - name: pull diag from s3 bucket if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} run: | @@ -1864,6 +2042,23 @@ jobs: path: | ${{ needs.setup.outputs.directory-path }}/diag* + Modinput-tests-report: + needs: run-modinput-tests + runs-on: ubuntu-latest + if: success() || failure() + steps: + - name: Download all summaries + uses: actions/download-artifact@v4 + with: + pattern: summary-modinput* + - name: Combine summaries into a table + run: | + echo "| Job | Total Tests | Passed | Failed | Errors| Skipped | Report Link" >> $GITHUB_STEP_SUMMARY + echo "| ---------- | ----------- | ------ | ------ | ------ | ------- | ------ |" >> $GITHUB_STEP_SUMMARY + for file in summary-modinput-*/summary.txt; do + cat "$file" >> $GITHUB_STEP_SUMMARY + done + run-scripted-input-tests-full-matrix: if: | ( !cancelled() && needs.build.result == 'success' && needs.test-inventory.outputs.scripted_inputs == 'true' ) && @@ -1891,7 +2086,7 @@ jobs: ARGO_NAMESPACE: ${{ needs.setup.outputs.argo-namespace }} SPLUNK_VERSION_BASE: ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} TEST_TYPE: "scripted_inputs" - permissions: + permissions: actions: read deployments: read contents: read @@ -2080,6 +2275,23 @@ jobs: name: splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} ${{ matrix.vendor-version.image }} ${{ steps.os-name-version.outputs.os-name }} ${{ steps.os-name-version.outputs.os-version }} test report path: "${{ needs.setup.outputs.directory-path }}/test-results/*.xml" reporter: java-junit + - name: Parse JUnit XML + if: ${{ !cancelled() }} + run: | + apt-get install -y libxml2-utils + junit_xml_path="${{ needs.setup.outputs.directory-path }}/test-results/*.xml" + total_tests=$(xmllint --xpath 'sum(//testsuite/@tests)' $junit_xml_path) + failures=$(xmllint --xpath 'sum(//testsuite/@failures)' $junit_xml_path) + errors=$(xmllint --xpath 'sum(//testsuite/@errors)' $junit_xml_path) + skipped=$(xmllint --xpath 'sum(//testsuite/@skipped)' $junit_xml_path) + passed=$((total_tests - failures - errors - skipped)) + echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ env.TEST_TYPE }} |$total_tests |$passed |$failures |$errors |${{steps.test_report.outputs.url_html}}" > summary.txt + - name: Upload-${{ matrix.splunk.version }}-${{ matrix.sc4s.version }}-${{ env.TEST_TYPE }} + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: summary-ko-${{ matrix.splunk.version }}-${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} + path: summary.txt - name: pull diag from s3 bucket if: ${{ failure() && steps.test_report.outputs.conclusion == 'failure' }} run: | @@ -2092,6 +2304,23 @@ jobs: path: | ${{ needs.setup.outputs.directory-path }}/diag* + scripted-input-tests-report: + needs: run-scripted-input-tests-full-matrix + runs-on: ubuntu-latest + if: success() || failure() + steps: + - name: Download all summaries + uses: actions/download-artifact@v4 + with: + pattern: summary-scripted-* + - name: Combine summaries into a table + run: | + echo "| Job | Total Tests | Passed | Failed | Errors| Skipped | Report Link" >> $GITHUB_STEP_SUMMARY + echo "| ---------- | ----------- | ------ | ------ | ------ | ------- | ------ |" >> $GITHUB_STEP_SUMMARY + for file in summary-ko*/summary.txt; do + cat "$file" >> $GITHUB_STEP_SUMMARY + done + pre-publish: if: ${{ !cancelled() && needs.validate-custom-version.result == 'success' }} # The following line will rename 'pre-publish' to 'pre-publish-not_main_pr' when PR is created towards main branch