Benchmarks #12
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This CI configuration for relative benchmarks is based on the research done | |
| # for scikit-image's implementation available here: | |
| # https://github.com/scikit-image/scikit-image/blob/9bdd010a8/.github/workflows/benchmarks.yml#L1 | |
| # Blog post with the rationale: https://labs.quansight.org/blog/2021/08/github-actions-benchmarks/ | |
| name: Benchmarks | |
| on: | |
| pull_request: | |
| types: [labeled] | |
| schedule: | |
| - cron: "6 6 * * 0" # every sunday | |
| workflow_dispatch: | |
| inputs: | |
| base_ref: | |
| description: "Baseline commit or git reference" | |
| required: true | |
| contender_ref: | |
| description: "Contender commit or git reference" | |
| required: true | |
| # This is the main configuration section that needs to be fine tuned to napari's needs | |
| # All the *_THREADS options is just to make the benchmarks more robust by not using parallelism | |
| env: | |
| OPENBLAS_NUM_THREADS: "1" | |
| MKL_NUM_THREADS: "1" | |
| OMP_NUM_THREADS: "1" | |
| ASV_OPTIONS: "--split --show-stderr --factor 1.5 --attribute timeout=900" | |
| # --split -> split final reports in tables | |
| # --show-stderr -> print tracebacks if errors occur | |
| # --factor 1.5 -> report anomaly if tested timings are beyond 1.5x base timings | |
| # --attribute timeout=300 -> override timeout attribute (default=60s) to allow slow tests to run | |
| # see https://asv.readthedocs.io/en/stable/commands.html#asv-continuous for more details! | |
| jobs: | |
| benchmark: | |
| if: ${{ github.event.label.name == 'run-benchmarks' && github.event_name == 'pull_request' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| name: ${{ matrix.benchmark-name }} | |
| runs-on: ${{ matrix.runs-on }} | |
| permissions: | |
| contents: read | |
| issues: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - benchmark-name: Qt | |
| asv-command: continuous | |
| selection-regex: "^benchmark_qt_.*" | |
| runs-on: macos-latest | |
| # Qt tests run on macOS to avoid using Xvfb business | |
| # xvfb makes everything run, but some tests segfault :shrug: | |
| # Fortunately, macOS graphics stack does not need xvfb! | |
| - benchmark-name: non-Qt | |
| asv-command: continuous | |
| selection-regex: "^benchmark_(?!qt_).*" | |
| runs-on: ubuntu-latest | |
| steps: | |
| # We need the full repo to avoid this issue | |
| # https://github.com/actions/checkout/issues/23 | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| name: Install Python | |
| with: | |
| python-version: "3.11" | |
| cache-dependency-path: pyproject.toml | |
| - uses: tlambert03/setup-qt-libs@v1 | |
| - name: Setup asv | |
| run: python -m pip install "asv[virtualenv]" | |
| env: | |
| PIP_CONSTRAINT: ${{ github.workspace }}/benchmarks/benchmark.txt | |
| - uses: octokit/[email protected] | |
| id: latest_release | |
| with: | |
| route: GET /repos/{owner}/{repo}/releases/latest | |
| owner: scverse | |
| repo: napari-spatialdata | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run ${{ matrix.benchmark-name }} benchmarks | |
| id: run_benchmark | |
| env: | |
| # asv will checkout commits, which might contain LFS artifacts; ignore those errors since | |
| # they are probably just documentation PNGs not needed here anyway | |
| GIT_LFS_SKIP_SMUDGE: 1 | |
| HEAD_LABEL: ${{ github.event.pull_request.head.label }} | |
| PIP_CONSTRAINT: ${{ github.workspace }}/benchmarks/benchmark.txt | |
| run: | | |
| set -euxo pipefail | |
| read -ra cmd_options <<< "$ASV_OPTIONS" | |
| # ID this runner | |
| asv machine --yes | |
| if [[ $GITHUB_EVENT_NAME == pull_request ]]; then | |
| EVENT_NAME="PR #${{ github.event.pull_request.number }}" | |
| BASE_REF=${{ github.event.pull_request.base.sha }} | |
| CONTENDER_REF=${GITHUB_SHA} | |
| echo "Baseline: ${BASE_REF} (${{ github.event.pull_request.base.label }})" | |
| echo "Contender: ${CONTENDER_REF} ($HEAD_LABEL)" | |
| elif [[ $GITHUB_EVENT_NAME == schedule ]]; then | |
| EVENT_NAME="cronjob" | |
| BASE_REF="${{ fromJSON(steps.latest_release.outputs.data).target_commitish }}" | |
| CONTENDER_REF="${GITHUB_SHA}" | |
| echo "Baseline: ${BASE_REF} (${{ fromJSON(steps.latest_release.outputs.data).tag_name }})" | |
| echo "Contender: ${CONTENDER_REF} (current main)" | |
| elif [[ $GITHUB_EVENT_NAME == workflow_dispatch ]]; then | |
| EVENT_NAME="manual trigger" | |
| BASE_REF="${{ github.event.inputs.base_ref }}" | |
| CONTENDER_REF="${{ github.event.inputs.contender_ref }}" | |
| echo "Baseline: ${BASE_REF} (workflow input)" | |
| echo "Contender: ${CONTENDER_REF} (workflow input)" | |
| fi | |
| echo "EVENT_NAME=$EVENT_NAME" >> "$GITHUB_ENV" | |
| echo "BASE_REF=$BASE_REF" >> "$GITHUB_ENV" | |
| echo "CONTENDER_REF=$CONTENDER_REF" >> "$GITHUB_ENV" | |
| # Run benchmarks for current commit against base | |
| asv continuous "${cmd_options[@]}" -b "${{ matrix.selection-regex }}" "${BASE_REF}" "${CONTENDER_REF}" \ | |
| | sed -E "/Traceback | failed$|PERFORMANCE DECREASED/ s/^/::error:: /" \ | |
| | tee asv_continuous.log | |
| # Report and export results for subsequent steps | |
| if grep "Traceback \|failed\|PERFORMANCE DECREASED" asv_continuous.log > /dev/null ; then | |
| exit 1 | |
| fi | |
| - name: Report Failures as Issue | |
| if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && failure() }} | |
| uses: JasonEtco/create-an-issue@v2 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PLATFORM: ${{ matrix.runs-on }} | |
| PYTHON: "3.9" | |
| BACKEND: ${{ matrix.benchmark-name }} | |
| RUN_ID: ${{ github.run_id }} | |
| TITLE: "[test-bot] Benchmark tests failing" | |
| with: | |
| filename: .github/BENCHMARK_FAIL_TEMPLATE.md | |
| update_existing: true | |
| - name: Add more info to artifact | |
| if: always() | |
| run: | | |
| # Copy the full `asv continuous` log | |
| cp asv_continuous.log .asv/results/asv_continuous_${{ matrix.benchmark-name }}.log | |
| # ensure that even if this isn't a PR, the benchmark_report workflow can run without error | |
| touch .asv/results/message_${{ matrix.benchmark-name }}.txt | |
| # Add the message that might be posted as a comment on the PR | |
| # We delegate the actual comment to `benchmarks_report.yml` due to | |
| # potential token permissions issues | |
| if [[ $GITHUB_EVENT_NAME == pull_request ]]; then | |
| echo "${{ github.event.pull_request.number }}" > .asv/results/pr_number | |
| echo \ | |
| "The ${{ matrix.benchmark-name }} benchmark run requested by $EVENT_NAME ($CONTENDER_REF vs $BASE_REF) has" \ | |
| "finished with status '${{ steps.run_benchmark.outcome }}'. See the" \ | |
| "[CI logs and artifacts](||BENCHMARK_CI_LOGS_URL||) for further details." \ | |
| > .asv/results/message_${{ matrix.benchmark-name }}.txt | |
| fi | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: asv-benchmark-results-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.benchmark-name }} | |
| path: .asv/results | |
| combine-artifacts: | |
| runs-on: ubuntu-latest | |
| needs: benchmark | |
| if: always() | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: asv-benchmark-results* | |
| path: asv_result | |
| merge-multiple: true | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: asv-benchmark-results-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} | |
| path: asv_result |