Introduce new public Reporter trait that is used from CLI types to report file status
#167
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance testing | |
| on: | |
| pull_request: | |
| paths: | |
| - 'stack-graphs/**' | |
| # In the event that there is a new push to the ref, cancel any running jobs because there are now obsolete, and wasting resources. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| BASE_REPO: ${{ github.event.pull_request.base.repo.owner.login }}/${{ github.event.pull_request.base.repo.name }} | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| BASE_DIR: base | |
| BASE_DB: base.sqlite | |
| BASE_ARTIFACT: base-perf-results | |
| HEAD_REPO: ${{ github.event.pull_request.head.repo.owner.login }}/${{ github.event.pull_request.head.repo.name }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| HEAD_DIR: head | |
| HEAD_DB: head.sqlite | |
| HEAD_ARTIFACT: head-perf-results | |
| TEST_NAME: typescript_benchmark | |
| MASSIF_OUT: perf.out | |
| MASSIF_REPORT: perf.txt | |
| TSSG_TS: tree-sitter-stack-graphs-typescript | |
| jobs: | |
| ## | |
| ## Determine relevant changes | |
| ## | |
| changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| base-sha: ${{ env.BASE_SHA }} | |
| head-sha: ${{ env.HEAD_SHA }} | |
| done: ${{ steps.done.outputs.cache-hit }} | |
| steps: | |
| - name: "Checkout base code" | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ env.BASE_REPO }} | |
| ref: ${{ env.BASE_SHA }} | |
| path: ${{ env.BASE_DIR }} | |
| fetch-depth: 0 | |
| lfs: false | |
| - name: Find last relevant base commit | |
| run: | | |
| printf 'BASE_SHA=%s\n' "$(git rev-list -1 ${{ env.BASE_SHA }} -- stack-graphs)" >> $GITHUB_ENV | |
| working-directory: ${{ env.BASE_DIR }} | |
| - name: "Checkout head code" | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ env.HEAD_REPO }} | |
| ref: ${{ env.HEAD_SHA }} | |
| path: ${{ env.HEAD_DIR }} | |
| fetch-depth: 0 | |
| lfs: false | |
| - name: "Find last relevant head commit" | |
| run: | | |
| printf 'HEAD_SHA=%s\n' "$(git rev-list -1 ${{ env.HEAD_SHA }} -- stack-graphs)" >> $GITHUB_ENV | |
| working-directory: ${{ env.HEAD_DIR }} | |
| - name: "Check cached status" | |
| id: done | |
| uses: actions/cache/restore@v3 | |
| with: | |
| path: done | |
| key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} | |
| lookup-only: 'true' | |
| ## | |
| ## Base performance | |
| ## | |
| base-perf: | |
| runs-on: ubuntu-latest | |
| needs: changes | |
| if: needs.changes.outputs.done != 'true' | |
| env: | |
| BASE_SHA: ${{ needs.changes.outputs.base-sha }} | |
| steps: | |
| - name: Install Rust environment | |
| uses: hecrj/setup-rust-action@v1 | |
| with: | |
| rust-version: stable | |
| - name: Cache Rust dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| key: ${{ runner.OS }}-cargo-home | |
| - name: Install valgrind | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y valgrind | |
| - name: "Cache base result" | |
| id: cache-base-result | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ${{ env.MASSIF_OUT }} | |
| ${{ env.MASSIF_REPORT }} | |
| key: ${{ runner.os }}-perf-result-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.TEST_NAME }} | |
| - name: "Checkout base code" | |
| if: steps.cache-base-result.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ env.BASE_REPO }} | |
| ref: ${{ env.BASE_SHA }} | |
| path: ${{ env.BASE_DIR }} | |
| lfs: true | |
| - name: "Build base CLI" | |
| if: steps.cache-base-result.outputs.cache-hit != 'true' | |
| run: cargo build --package ${{ env.TSSG_TS }} --features cli --release | |
| working-directory: ${{ env.BASE_DIR }} | |
| env: | |
| CARGO_PROFILE_RELEASE_DEBUG: true | |
| - name: Checkout benchmark code | |
| if: steps.cache-base-result.outputs.cache-hit != 'true' | |
| run: | | |
| unzip ${{ env.TEST_NAME }}.zip | |
| working-directory: ${{ env.BASE_DIR }}/data | |
| - name: Profile base memory | |
| if: steps.cache-base-result.outputs.cache-hit != 'true' | |
| run: | | |
| valgrind \ | |
| --tool=massif \ | |
| --massif-out-file=${{ env.MASSIF_OUT }} \ | |
| ${{ env.BASE_DIR }}/target/release/${{ env.TSSG_TS }} \ | |
| index -D ${{ env.BASE_DB }} --max-file-time=30 --hide-error-details -- \ | |
| ${{ env.BASE_DIR }}/data/${{ env.TEST_NAME }} | |
| ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }} | |
| - name: Upload results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.BASE_ARTIFACT }} | |
| path: | | |
| ${{ env.MASSIF_OUT }} | |
| ${{ env.MASSIF_REPORT }} | |
| ## | |
| ## Head performance | |
| ## | |
| head-perf: | |
| runs-on: ubuntu-latest | |
| needs: changes | |
| if: needs.changes.outputs.done != 'true' | |
| env: | |
| HEAD_SHA: ${{ needs.changes.outputs.head-sha }} | |
| steps: | |
| - name: Install Rust environment | |
| uses: hecrj/setup-rust-action@v1 | |
| with: | |
| rust-version: stable | |
| - name: Cache Rust dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo | |
| key: ${{ runner.OS }}-cargo-home | |
| - name: Install valgrind | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y valgrind | |
| - name: "Cache head result" | |
| id: cache-head-result | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ${{ env.MASSIF_OUT }} | |
| ${{ env.MASSIF_REPORT }} | |
| key: ${{ runner.os }}-perf-result-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} | |
| - name: "Checkout head code" | |
| if: steps.cache-head-result.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ env.HEAD_REPO }} | |
| ref: ${{ env.HEAD_SHA }} | |
| path: ${{ env.HEAD_DIR }} | |
| lfs: true | |
| - name: "Build head CLI" | |
| if: steps.cache-head-result.outputs.cache-hit != 'true' | |
| run: cargo build --package ${{ env.TSSG_TS }} --features cli --release | |
| working-directory: ${{ env.HEAD_DIR }} | |
| env: | |
| CARGO_PROFILE_RELEASE_DEBUG: true | |
| - name: Checkout benchmark code | |
| if: steps.cache-head-result.outputs.cache-hit != 'true' | |
| run: | | |
| unzip ${{ env.TEST_NAME }}.zip | |
| working-directory: ${{ env.HEAD_DIR }}/data | |
| - name: Profile head memory | |
| if: steps.cache-head-result.outputs.cache-hit != 'true' | |
| run: | | |
| valgrind \ | |
| --tool=massif \ | |
| --massif-out-file=${{ env.MASSIF_OUT }} \ | |
| ${{ env.HEAD_DIR }}/target/release/${{ env.TSSG_TS }} \ | |
| index -D ${{ env.HEAD_DB }} --max-file-time=30 --hide-error-details -- \ | |
| ${{ env.HEAD_DIR }}/data/${{ env.TEST_NAME }} | |
| ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }} | |
| - name: Upload results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.HEAD_ARTIFACT }} | |
| path: | | |
| ${{ env.MASSIF_OUT }} | |
| ${{ env.MASSIF_REPORT }} | |
| ## | |
| ## Performance summary | |
| ## | |
| perf-summary: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - changes | |
| - base-perf | |
| - head-perf | |
| if: needs.changes.outputs.done != 'true' | |
| env: | |
| BASE_SHA: ${{ needs.changes.outputs.base-sha }} | |
| HEAD_SHA: ${{ needs.changes.outputs.head-sha }} | |
| SRC_DIR: src | |
| COMMENT_JSON: perf-summary.json | |
| steps: | |
| # | |
| # Install tools | |
| # | |
| - name: Install valgrind | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y valgrind | |
| # | |
| # Download results | |
| # | |
| - name: Download base results | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.BASE_ARTIFACT }} | |
| path: ${{ env.BASE_ARTIFACT }} | |
| - name: Download head results | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.HEAD_ARTIFACT }} | |
| path: ${{ env.HEAD_ARTIFACT }} | |
| # | |
| # Create report | |
| # | |
| - name: "Checkout code" | |
| uses: actions/checkout@v3 | |
| with: | |
| path: ${{ env.SRC_DIR }} | |
| - name: Generate summary | |
| run: | | |
| ${{ env.SRC_DIR }}/script/ci-perf-summary-md \ | |
| ${{ env.BASE_ARTIFACT }}/${{ env.MASSIF_OUT }} \ | |
| ${{ env.HEAD_ARTIFACT }}/${{ env.MASSIF_OUT }} \ | |
| 'Comparing base ${{ env.BASE_REPO }}@${{ env.BASE_SHA }} with head ${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }} on ${{ env.TEST_NAME }} benchmark. For details see [workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) artifacts. _Note that performance is tested on the last commits with changes in `stack-graphs`, not on every commit._' \ | |
| | ${{ env.SRC_DIR }}/script/ci-comment-json > ${{ env.COMMENT_JSON }} | |
| - name: Add summary comment to PR | |
| run: | | |
| curl \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ github.token }}" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| ${{ github.event.pull_request.comments_url }} \ | |
| -d '@${{ env.COMMENT_JSON }}' | |
| - name: Create status marker | |
| run: touch done | |
| - name: "Cache status" | |
| uses: actions/cache/save@v3 | |
| with: | |
| path: done | |
| key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} |