Skip to content

Commit b718493

Browse files
robert3005robert3005
committed
Convert compress benchmark to a binary
1 parent 3883c7a commit b718493

16 files changed

+595
-348
lines changed

‎.github/workflows/bench-pr.yml

+2-75
Original file line numberDiff line numberDiff line change
@@ -56,81 +56,6 @@ jobs:
5656
run: cargo codspeed run
5757
token: ${{ secrets.CODSPEED_TOKEN }}
5858

59-
bench-criterion:
60-
needs: label_trigger
61-
strategy:
62-
matrix:
63-
benchmark:
64-
- id: compress
65-
name: Vortex Compression
66-
runs-on: [ self-hosted, gcp ]
67-
if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }}
68-
steps:
69-
- uses: actions/checkout@v4
70-
- uses: ./.github/actions/cleanup
71-
- uses: ./.github/actions/setup-rust
72-
73-
# The compression benchmarks rely on DuckDB being installed to convert CSV to Parquet
74-
- name: Install DuckDB
75-
uses: opt-nc/[email protected]
76-
if: runner.environment != 'self-hosted'
77-
with:
78-
version: v1.0.0
79-
80-
- name: Set tempdir
81-
if: runner.environment == 'self-hosted'
82-
run: |
83-
echo "TMPDIR=/work" >> $GITHUB_ENV
84-
85-
- name: Run benchmark
86-
shell: bash
87-
env:
88-
BENCH_VORTEX_RATIOS: '.*'
89-
RUSTFLAGS: '-C target-cpu=native'
90-
run: |
91-
cargo install cargo-criterion
92-
sudo apt-get update && sudo apt-get install -y jq
93-
94-
cargo criterion \
95-
--bench ${{ matrix.benchmark.id }} \
96-
--features mimalloc \
97-
--message-format=json \
98-
> ${{ matrix.benchmark.id }}-raw.json
99-
100-
cat ${{ matrix.benchmark.id }}-raw.json \
101-
| bash scripts/coerce-criterion-json.sh \
102-
> ${{ matrix.benchmark.id }}.json
103-
104-
- name: Setup AWS CLI
105-
uses: aws-actions/configure-aws-credentials@v4
106-
with:
107-
role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
108-
aws-region: us-east-1
109-
- name: Install uv
110-
uses: astral-sh/setup-uv@v5
111-
- name: Compare results
112-
shell: bash
113-
run: |
114-
set -Eeu -o pipefail -x
115-
116-
base_commit_sha=${{ github.event.pull_request.base.sha }}
117-
118-
aws s3 cp s3://vortex-benchmark-results-database/data.json - \
119-
| grep $base_commit_sha \
120-
> base.json
121-
122-
echo '# Benchmarks: ${{ matrix.benchmark.id }}' > comment.md
123-
echo '<details>' >> comment.md
124-
echo '<summary>Table of Results</summary>' >> comment.md
125-
echo '' >> comment.md
126-
uv run scripts/compare-benchmark-jsons.py base.json ${{ matrix.benchmark.id }}.json \
127-
>> comment.md
128-
echo '</details>' >> comment.md
129-
- name: Comment PR
130-
uses: thollander/actions-comment-pull-request@v3
131-
with:
132-
file-path: comment.md
133-
comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
13459
bench:
13560
needs: label_trigger
13661
runs-on: [ self-hosted, gcp ]
@@ -139,6 +64,8 @@ jobs:
13964
benchmark:
14065
- id: random_access
14166
name: Random Access
67+
- id: compress
68+
name: Compression
14269
if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }}
14370
steps:
14471
- uses: actions/checkout@v4

‎.github/workflows/bench.yml

+2-52
Original file line numberDiff line numberDiff line change
@@ -62,65 +62,15 @@ jobs:
6262
run: cargo codspeed run
6363
token: ${{ secrets.CODSPEED_TOKEN }}
6464

65-
bench-criterion:
66-
strategy:
67-
matrix:
68-
benchmark:
69-
- id: compress
70-
name: Vortex Compression
71-
runs-on: [ self-hosted, gcp ]
72-
steps:
73-
- uses: actions/checkout@v4
74-
- uses: ./.github/actions/cleanup
75-
- uses: ./.github/actions/setup-rust
76-
77-
# The compression benchmarks rely on DuckDB being installed to convert CSV to Parquet
78-
- name: Install DuckDB
79-
uses: opt-nc/[email protected]
80-
if: runner.environment != 'self-hosted'
81-
with:
82-
version: v1.0.0
83-
84-
- name: Set tempdir
85-
if: runner.environment == 'self-hosted'
86-
run: |
87-
echo "TMPDIR=/work" >> $GITHUB_ENV
88-
89-
- name: Run benchmark
90-
shell: bash
91-
env:
92-
BENCH_VORTEX_RATIOS: '.*'
93-
RUSTFLAGS: '-C target-cpu=native'
94-
run: |
95-
cargo install cargo-criterion
96-
sudo apt-get update && sudo apt-get install -y jq
97-
98-
cargo criterion \
99-
--bench ${{ matrix.benchmark.id }} \
100-
--features mimalloc \
101-
--message-format=json \
102-
> ${{ matrix.benchmark.id }}-raw.json
103-
104-
cat ${{ matrix.benchmark.id }}-raw.json \
105-
| bash scripts/coerce-criterion-json.sh \
106-
> ${{ matrix.benchmark.id }}.json
107-
108-
- name: Setup AWS CLI
109-
uses: aws-actions/configure-aws-credentials@v4
110-
with:
111-
role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
112-
aws-region: us-east-1
113-
- name: Upload Benchmark Results
114-
shell: bash
115-
run: |
116-
bash scripts/cat-s3.sh vortex-benchmark-results-database data.json ${{ matrix.benchmark.id }}.json
11765
bench:
11866
runs-on: [ self-hosted, gcp ]
11967
strategy:
12068
matrix:
12169
benchmark:
12270
- id: random_access
12371
name: Random Access
72+
- id: compress
73+
name: Compression
12474
steps:
12575
- uses: actions/checkout@v4
12676
- uses: ./.github/actions/cleanup

‎.github/workflows/sql-benchmarks.yml

-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ on:
1010
jobs:
1111
bench:
1212
strategy:
13-
fail-fast: false
1413
matrix:
1514
# Regarding "include:":
1615
# https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow#example-adding-configurations

‎bench-vortex/benches/compress.rs

+1-5
Original file line numberDiff line numberDiff line change
@@ -136,10 +136,6 @@ fn vortex_decompress_read(runtime: &Runtime, buf: Bytes) -> VortexResult<Vec<Arr
136136
})
137137
}
138138

139-
fn vortex_compressed_written_size(runtime: &Runtime, array: &Array) -> VortexResult<u64> {
140-
vortex_compress_write(runtime, array, &mut Vec::new())
141-
}
142-
143139
fn benchmark_compress<F, U>(
144140
c: &mut Criterion,
145141
make_uncompressed: F,
@@ -177,7 +173,7 @@ fn benchmark_compress<F, U>(
177173
group.bench_function(bench_name, |b| {
178174
b.iter_with_large_drop(|| {
179175
compressed_size =
180-
vortex_compressed_written_size(runtime, uncompressed.as_ref()).unwrap();
176+
vortex_compress_write(runtime, uncompressed.as_ref(), &mut Vec::new()).unwrap();
181177
});
182178
});
183179
group.finish();

‎bench-vortex/src/bench_run.rs

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
use std::future::Future;
2+
use std::hint::black_box;
3+
use std::time::{Duration, Instant};
4+
5+
use tokio::runtime::Runtime;
6+
7+
pub fn run<O, R, F>(runtime: &Runtime, iterations: usize, mut routine: R) -> Duration
8+
where
9+
R: FnMut() -> F,
10+
F: Future<Output = O>,
11+
{
12+
run_with_setup(runtime, iterations, || (), |_| routine())
13+
}
14+
15+
pub fn run_with_setup<I, O, S, R, F>(
16+
runtime: &Runtime,
17+
iterations: usize,
18+
mut setup: S,
19+
mut routine: R,
20+
) -> Duration
21+
where
22+
S: FnMut() -> I,
23+
R: FnMut(I) -> F,
24+
F: Future<Output = O>,
25+
{
26+
for _ in 0..2 {
27+
black_box(routine(setup()));
28+
}
29+
30+
let mut fastest_result = Duration::from_millis(u64::MAX);
31+
for _ in 0..iterations {
32+
let state = black_box(setup());
33+
let elapsed = runtime.block_on(async {
34+
let start = Instant::now();
35+
let output = routine(state).await;
36+
let elapsed = start.elapsed();
37+
drop(black_box(output));
38+
elapsed
39+
});
40+
fastest_result = fastest_result.min(elapsed);
41+
}
42+
43+
fastest_result
44+
}

0 commit comments

Comments
 (0)