Valgrind #560
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Valgrind | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| test-file: | |
| description: 'new_test/<test-file>' | |
| required: false | |
| default: "" | |
| use-server-rc: | |
| type: boolean | |
| description: 'Use server release candidate?' | |
| required: true | |
| default: false | |
| server-tag: | |
| required: false | |
| default: latest | |
| massif: | |
| type: boolean | |
| description: 'Use massif for testing memory usage' | |
| required: false | |
| default: false | |
| env: | |
| PYTHON_TAG: cp38 | |
| jobs: | |
| look-for-wheel-in-jfrog: | |
| outputs: | |
| num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} | |
| # So we can pass the python tag to a reusable workflow | |
| python-tag: ${{ env.PYTHON_TAG }} | |
| runs-on: ubuntu-22.04 | |
| env: | |
| JF_SEARCH_RESULTS_FILE_NAME: wheel_commit_matches.txt | |
| steps: | |
| - uses: jfrog/setup-jfrog-cli@v4 | |
| env: | |
| JF_URL: ${{ secrets.JFROG_PLATFORM_URL }} | |
| JF_ACCESS_TOKEN: ${{ secrets.JFROG_ACCESS_TOKEN }} | |
| - name: Get shortened commit hash of this workflow run | |
| # versioningit commit sha is always 8 chars long it seems | |
| run: echo SHORT_GITHUB_SHA=$(echo ${{ github.sha }} | cut -c1-8) >> $GITHUB_ENV | |
| - name: Look for wheel built with default settings in JFrog | |
| # AQL has the option to exclude patterns from search results | |
| # but it doesn't allow regex, so we can't filter out any type of label in a wheel name | |
| # Example: we want to filter out "unoptimized" and "dsym" but in case we add more labels, we want to use regex | |
| # to handle those new labels without updating the regex. | |
| run: jf rt search "${{ vars.JFROG_GENERIC_REPO_NAME }}/${{ github.ref_name }}/*${{ env.SHORT_GITHUB_SHA }}*${{ env.PYTHON_TAG }}*manylinux*x86_64*.whl" > ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | |
| - name: Show unfiltered results | |
| run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | |
| - name: Install sponge | |
| run: sudo apt install -y moreutils | |
| - name: Filter out wheels with labels in results | |
| run: jq 'map(select(.path | test("${{ env.SHORT_GITHUB_SHA }}\\.") | not))' ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | sponge ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | |
| shell: bash | |
| - name: Check if artifacts with labels were filtered out | |
| run: cat ${{ env.JF_SEARCH_RESULTS_FILE_NAME }} | |
| - name: Count artifacts | |
| id: count_num_artifacts_found | |
| run: echo num_artifacts=$(jq length ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) >> $GITHUB_OUTPUT | |
| - name: Multiple artifacts found, not sure which one to use. Fail out | |
| if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts > 1 }} | |
| run: exit 1 | |
| - name: Found the exact artifact in JFrog. Get the artifact name | |
| if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} | |
| run: echo ARTIFACT_PATH=$(jq -r .[0].path ${{ env.JF_SEARCH_RESULTS_FILE_NAME }}) >> $GITHUB_ENV | |
| - name: Then download artifact from JFrog | |
| if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} | |
| run: jf rt download --flat --fail-no-op ${{ env.ARTIFACT_PATH }} | |
| - name: Pass to valgrind job | |
| if: ${{ steps.count_num_artifacts_found.outputs.num_artifacts == 1 }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| # Artifact name doesn't matter. Valgrind job downloads all artifacts to get the one wheel | |
| if-no-files-found: error | |
| path: './*.whl' | |
| build-manylinux-wheel: | |
| needs: look-for-wheel-in-jfrog | |
| if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} | |
| uses: ./.github/workflows/build-wheels.yml | |
| with: | |
| python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' | |
| platform-tag: manylinux_x86_64 | |
| sha-to-build-and-test: ${{ github.sha }} | |
| secrets: inherit | |
| upload-built-wheel-to-jfrog: | |
| needs: build-manylinux-wheel | |
| # TODO: this job should skip when this workflow is run on central branches | |
| # We already have artifacts available for central branches in the PyPI-type JFrog repo | |
| # The problem is we have to conditionally skip this job, but using the github context to get the branch name | |
| # doesn't work for some reason. Just leave this alone for now. | |
| uses: ./.github/workflows/upload-to-jfrog.yml | |
| with: | |
| jfrog-repo-name: ${{ vars.JFROG_GENERIC_REPO_NAME }} | |
| secrets: inherit | |
| valgrind: | |
| env: | |
| MASSIF_REPORT_FILE_NAME: massif.out | |
| needs: [ | |
| look-for-wheel-in-jfrog, | |
| build-manylinux-wheel | |
| ] | |
| # Case 1: Found artifact in JFrog | |
| # Case 2: Did not find artifact in JFrog, had to build it in GHA | |
| if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'success' && (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 1) || (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 && needs.build-manylinux-wheel.result == 'success')) }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Convert Python tag to Python version | |
| run: echo PYTHON_VERSION=$(echo ${{ env.PYTHON_TAG }} | sed -e "s/cp3/cp3./" -e "s/cp//") >> $GITHUB_ENV | |
| shell: bash | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '${{ env.PYTHON_VERSION }}' | |
| architecture: 'x64' | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| merge-multiple: true | |
| - name: Install client | |
| run: pip install ./*.whl | |
| - name: Install test dependencies | |
| run: pip install -r test/requirements.txt | |
| - name: Run EE server | |
| uses: ./.github/actions/run-ee-server | |
| with: | |
| use-server-rc: ${{ inputs.use-server-rc }} | |
| server-tag: ${{ inputs.server-tag }} | |
| docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }} | |
| docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }} | |
| - run: sudo apt update | |
| - run: sudo apt install valgrind -y | |
| - run: echo VALGRIND_ARGS="--tool=massif --massif-out-file=./${{ env.MASSIF_REPORT_FILE_NAME }}" >> $GITHUB_ENV | |
| if: ${{ inputs.massif }} | |
| - run: echo VALGRIND_ARGS="--leak-check=full" >> $GITHUB_ENV | |
| if: ${{ !inputs.massif }} | |
| - run: PYTHONMALLOC=malloc valgrind --error-exitcode=1 ${{ env.VALGRIND_ARGS }} python3 -m pytest -v new_tests/${{ github.event.inputs.test-file }} | |
| working-directory: test | |
| # TODO: upload report as artifact | |
| - run: ms_print ./${{ env.MASSIF_REPORT_FILE_NAME }} | |
| if: ${{ !cancelled() && inputs.massif }} | |
| working-directory: test | |
| # See reason for deleting artifacts in dev-workflow-p2.yml | |
| delete-artifacts: | |
| needs: [ | |
| # These jobs must have downloaded the artifact from Github before we can delete it | |
| upload-built-wheel-to-jfrog, | |
| valgrind | |
| ] | |
| # Workflow run must clean up after itself even if cancelled | |
| if: ${{ always() }} | |
| uses: ./.github/workflows/delete-artifacts.yml |