diff --git a/.github/workflows/hyperfine.yml b/.github/workflows/hyperfine.yml index 87bf26f3e3..61a4435974 100644 --- a/.github/workflows/hyperfine.yml +++ b/.github/workflows/hyperfine.yml @@ -2,7 +2,7 @@ name: Hyperfine Benchmark on: pull_request: - branches: [ '**' ] + branches: ["**"] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -11,11 +11,14 @@ concurrency: env: CARGO_TERM_COLOR: always +permissions: + issues: write + jobs: build-programs: strategy: matrix: - branch: [ base, head ] + branch: [base, head] name: Build Cairo programs for ${{ matrix.branch }} runs-on: ubuntu-22.04 outputs: @@ -39,7 +42,7 @@ jobs: if: ${{ steps.cache.outputs.cache-hit != 'true' }} uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: "3.9" - name: Install Cairo compiler if: ${{ steps.cache.outputs.cache-hit != 'true' }} @@ -54,20 +57,18 @@ jobs: - name: Export benchmark hashes id: export-hashes - run: - echo "benchmark-hashes-${{ matrix.branch }}=${{ hashFiles( 'cairo_programs/benchmarks/*.cairo' ) }}" >> "$GITHUB_OUTPUT" - + run: echo "benchmark-hashes-${{ matrix.branch }}=${{ hashFiles( 'cairo_programs/benchmarks/*.cairo' ) }}" >> "$GITHUB_OUTPUT" build-binaries: strategy: matrix: - branch: [ base, head ] + branch: [base, head] name: Build cairo-vm-cli for ${{ matrix.branch }} runs-on: ubuntu-22.04 steps: - name: Populate cache uses: actions/cache@v3 - id: cache + id: cache with: path: bin/cairo-vm-cli-${{ matrix.branch }} key: binary-${{ github.event.pull_request[matrix.branch].sha }} @@ -93,106 +94,105 @@ jobs: mkdir bin cp target/release/cairo-vm-cli bin/cairo-vm-cli-${{ matrix.branch }} - run-hyperfine: strategy: matrix: - program_state: [ modified, unmodified ] + program_state: [modified, unmodified] name: Run benchmarks for ${{ matrix.program_state }} programs - needs: [ build-programs, build-binaries ] + needs: [build-programs, build-binaries] runs-on: ubuntu-22.04 steps: - - name: Install Hyperfine - uses: taiki-e/install-action@v2 - with: - tool: hyperfine@1.16 - - - name: Fetch base binary - uses: actions/cache/restore@v3 - with: - path: bin/cairo-vm-cli-base - key: binary-${{ github.event.pull_request.base.sha }} - - - name: Fetch HEAD binary - uses: actions/cache/restore@v3 - with: - path: bin/cairo-vm-cli-head - key: binary-${{ github.event.pull_request.head.sha }} - - - name: Fetch base programs - uses: actions/cache/restore@v3 - with: - path: base_programs/*.json - key: benchmarks-base-${{ needs.build-programs.outputs.benchmark-hashes-base }} - - - name: Fetch head programs - uses: actions/cache/restore@v3 - with: - path: head_programs/*.json - key: benchmarks-head-${{ needs.build-programs.outputs.benchmark-hashes-head }} - - - name: Benchmark ${{ matrix.program_state }} programs - id: run-benchmarks - run: | - sudo swapoff -a - mkdir target_programs - if [ 'modified' = ${{ matrix.program_state }} ]; then - BINS=head - for f in head_programs/*.json; do - # Only run new or modified benchmarks - if ! cmp -s ${f/head/base} $f; then - cp $f target_programs/ - fi - done - else - BINS="base,head" - for f in base_programs/*.json; do - # Only run unmodified benchmarks - if cmp -s ${f/base/head} $f; then - cp $f target_programs/ - fi - done - fi - find target_programs -name '*.json' -exec basename -s .json '{}' '+' | \ - sort | xargs -I '{program}' \ - hyperfine -N -r 10 --export-markdown "target_programs/{program}.md" \ - -L bin "$BINS" -n "{bin} {program}" \ - -s "cat ./bin/cairo-vm-cli-{bin} target_programs/{program}.json" \ - "./bin/cairo-vm-cli-{bin} --proof_mode --layout starknet_with_keccak \ - --memory_file /dev/null --trace_file /dev/null target_programs/{program}.json" - echo "benchmark_count=$(ls target_programs/*.md | wc -l)" >> $GITHUB_OUTPUT - - - name: Print tables - if: steps.run-benchmarks.outputs.benchmark_count != 0 - run: | - { - echo "Benchmark Results for ${{ matrix.program_state }} programs :rocket:" - for f in target_programs/*.md; do - echo - cat $f - done - } | tee -a comment_body.md - - - name: Find comment - if: ${{ steps.run-benchmarks.outputs.benchmark_count != 0 }} - uses: peter-evans/find-comment@v2 - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: Benchmark Results for ${{ matrix.program_state }} programs - - - name: Create comment - if: steps.fc.outputs.comment-id == '' && steps.run-benchmarks.outputs.benchmark_count != 0 - uses: peter-evans/create-or-update-comment@v3 - with: - issue-number: ${{ github.event.pull_request.number }} - body-path: comment_body.md - - - name: Update comment - if: steps.fc.outputs.comment-id != '' && steps.run-benchmarks.outputs.benchmark_count != 0 - uses: peter-evans/create-or-update-comment@v3 - with: - comment-id: ${{ steps.fc.outputs.comment-id }} - body-path: comment_body.md - edit-mode: replace + - name: Install Hyperfine + uses: taiki-e/install-action@v2 + with: + tool: hyperfine@1.16 + + - name: Fetch base binary + uses: actions/cache/restore@v3 + with: + path: bin/cairo-vm-cli-base + key: binary-${{ github.event.pull_request.base.sha }} + + - name: Fetch HEAD binary + uses: actions/cache/restore@v3 + with: + path: bin/cairo-vm-cli-head + key: binary-${{ github.event.pull_request.head.sha }} + + - name: Fetch base programs + uses: actions/cache/restore@v3 + with: + path: base_programs/*.json + key: benchmarks-base-${{ needs.build-programs.outputs.benchmark-hashes-base }} + + - name: Fetch head programs + uses: actions/cache/restore@v3 + with: + path: head_programs/*.json + key: benchmarks-head-${{ needs.build-programs.outputs.benchmark-hashes-head }} + + - name: Benchmark ${{ matrix.program_state }} programs + id: run-benchmarks + run: | + sudo swapoff -a + mkdir target_programs + if [ 'modified' = ${{ matrix.program_state }} ]; then + BINS=head + for f in head_programs/*.json; do + # Only run new or modified benchmarks + if ! cmp -s ${f/head/base} $f; then + cp $f target_programs/ + fi + done + else + BINS="base,head" + for f in base_programs/*.json; do + # Only run unmodified benchmarks + if cmp -s ${f/base/head} $f; then + cp $f target_programs/ + fi + done + fi + find target_programs -name '*.json' -exec basename -s .json '{}' '+' | \ + sort | xargs -I '{program}' \ + hyperfine -N -r 10 --export-markdown "target_programs/{program}.md" \ + -L bin "$BINS" -n "{bin} {program}" \ + -s "cat ./bin/cairo-vm-cli-{bin} target_programs/{program}.json" \ + "./bin/cairo-vm-cli-{bin} --proof_mode --layout starknet_with_keccak \ + --memory_file /dev/null --trace_file /dev/null target_programs/{program}.json" + echo "benchmark_count=$(ls target_programs/*.md | wc -l)" >> $GITHUB_OUTPUT + + - name: Print tables + if: steps.run-benchmarks.outputs.benchmark_count != 0 + run: | + { + echo "Benchmark Results for ${{ matrix.program_state }} programs :rocket:" + for f in target_programs/*.md; do + echo + cat $f + done + } | tee -a comment_body.md + + - name: Find comment + if: ${{ steps.run-benchmarks.outputs.benchmark_count != 0 }} + uses: peter-evans/find-comment@v2 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: Benchmark Results for ${{ matrix.program_state }} programs + + - name: Create comment + if: steps.fc.outputs.comment-id == '' && steps.run-benchmarks.outputs.benchmark_count != 0 + uses: peter-evans/create-or-update-comment@v3 + with: + issue-number: ${{ github.event.pull_request.number }} + body-path: comment_body.md + + - name: Update comment + if: steps.fc.outputs.comment-id != '' && steps.run-benchmarks.outputs.benchmark_count != 0 + uses: peter-evans/create-or-update-comment@v3 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + body-path: comment_body.md + edit-mode: replace