Benchmarks in CI (#485)

.github/runners/Dockerfile

@@ -4,7 +4,8 @@ USER root
 RUN apt update && \
     apt install -y \
     build-essential \
-    curl
+    curl \
+    git
 
 USER runner
 CMD /bin/bash

.github/workflows/benchmarks-pages.yaml

@@ -0,0 +1,35 @@
+name: 
+
+on:
+  push:
+    branches:
+      - dev
+
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+jobs:
+  run-avx-bench:
+    runs-on: avx
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly-2024-01-04
+      - name: Run benchmark
+        run: ./scripts/bench.sh -- --output-format bencher | tee output.txt
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'cargo'
+          output-file-path: output.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true

.github/workflows/ci.yaml

@@ -3,10 +3,7 @@ name: CI
 on:
   push:
     branches:
-      - main
-      - main-v[0-9].**
-    tags:
-      - v[0-9].**
+      - dev
 
   pull_request:
     types:
@@ -51,6 +48,32 @@ jobs:
           toolchain: nightly-2024-01-04
       - uses: Swatinem/rust-cache@v2
       - run: ./scripts/test_avx.sh
+
+  run-avx-bench:
+    runs-on: avx
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly-2024-01-04
+      - name: Run benchmark
+        run: ./scripts/bench.sh -- --output-format bencher | tee output.txt
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'cargo'
+          output-file-path: output.txt
+          external-data-json-path: ./cache/benchmark-data.json
+          fail-on-alert: true
+          summary-always: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+          alert-comment-cc-users: '@spapinistarkware'
 
   run-tests:
     runs-on: ubuntu-latest

Cargo.toml

@@ -19,6 +19,9 @@ bytemuck = { version = "1.14.3", features = ["derive"] }
 criterion = { version = "0.5.1", features = ["html_reports"] }
 rand = { version = "0.8.5", features = ["small_rng"] }
 
+[lib]
+bench = false
+
 [lints.rust]
 warnings = "deny"
 future-incompatible = "deny"

benches/bit_rev.rs

@@ -6,12 +6,12 @@ use criterion::Criterion;
 pub fn cpu_bit_rev(c: &mut criterion::Criterion) {
     use stwo::core::fields::m31::BaseField;
 
-    const SIZE: usize = 1 << 28;
+    const SIZE: usize = 1 << 24;
     let mut data: Vec<_> = (0..SIZE as u32)
         .map(BaseField::from_u32_unchecked)
         .collect();
 
-    c.bench_function("cpu bit_rev", |b| {
+    c.bench_function("cpu bit_rev 24bit", |b| {
         b.iter(|| {
             stwo::core::utils::bit_reverse(&mut data);
         })
@@ -29,7 +29,7 @@ pub fn avx512_bit_rev(c: &mut criterion::Criterion) {
         return;
     }
 
-    const SIZE: usize = 1 << 28;
+    const SIZE: usize = 1 << 26;
     let data: Vec<_> = (0..SIZE as u32)
         .map(BaseField::from_u32_unchecked)
         .collect();
@@ -39,7 +39,7 @@ pub fn avx512_bit_rev(c: &mut criterion::Criterion) {
         .map(PackedBaseField::from_array)
         .collect();
 
-    c.bench_function("avx bit_rev", |b| {
+    c.bench_function("avx bit_rev 26bit", |b| {
         b.iter(|| {
             bit_reverse_m31(cast_slice_mut(&mut data[..]));
         })

benches/fft.rs

@@ -13,7 +13,7 @@ pub fn avx512_ifft(c: &mut criterion::Criterion) {
         return;
     }
 
-    const LOG_SIZE: u32 = 28;
+    const LOG_SIZE: u32 = 26;
     let domain = CanonicCoset::new(LOG_SIZE).circle_domain();
     let values = (0..domain.size())
         .map(|i| BaseField::from_u32_unchecked(i as u32))
@@ -23,7 +23,7 @@ pub fn avx512_ifft(c: &mut criterion::Criterion) {
     let mut values = BaseFieldVec::from_iter(values);
     let twiddle_dbls = get_itwiddle_dbls(domain.half_coset);
 
-    c.bench_function("avx ifft", |b| {
+    c.bench_function("avx ifft 26bit", |b| {
         b.iter(|| unsafe {
             ifft::ifft(
                 std::mem::transmute(values.data.as_mut_ptr()),

benches/field.rs

@@ -1,3 +1,4 @@
+use criterion::Criterion;
 use rand::rngs::ThreadRng;
 use rand::Rng;
 use stwo::core::fields::cm31::CM31;
@@ -187,17 +188,21 @@ pub fn avx512_m31_operations_bench(c: &mut criterion::Criterion) {
 
 #[cfg(target_arch = "x86_64")]
 criterion::criterion_group!(
-    m31_benches,
-    m31_operations_bench,
-    avx512_m31_operations_bench
+    name=m31_benches;
+    config = Criterion::default().sample_size(10);
+    targets=
+        m31_operations_bench,
+        avx512_m31_operations_bench
 );
 #[cfg(not(target_arch = "x86_64"))]
 criterion::criterion_group!(m31_benches, m31_operations_bench);
 
 criterion::criterion_group!(
-    field_comparison,
-    m31_operations_bench,
-    cm31_operations_bench,
-    qm31_operations_bench
+    name=field_comparison;
+    config = Criterion::default().sample_size(10);
+    targets=
+        m31_operations_bench,
+        cm31_operations_bench,
+        qm31_operations_bench
 );
 criterion::criterion_main!(field_comparison, m31_benches);

scripts/bench.sh

@@ -2,4 +2,4 @@
 # Can be used as a drop in replacement for `cargo bench`.
 # For example, `./scripts/bench.sh` will run all benchmarks.
 # or `./scripts/bench.sh M31` will run only the M31 benchmarks.
-RUSTFLAGS="-Awarnings -C target-cpu=native -C target-feature=+avx512f -C opt-level=2" cargo bench "$@"
+RUSTFLAGS="-Awarnings -C target-cpu=native -C target-feature=+avx512f -C opt-level=2" cargo bench $@