From c7f0c2d1759eab0b3be4facc3f31537d8ac01f98 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Sat, 10 Aug 2024 18:04:33 +0800 Subject: [PATCH 01/15] chore: add codspeed for benchmark --- .github/workflows/test_benchmark.yml | 56 ++++++++++++++++++++++++++ core/Cargo.lock | 35 ++++++++++++++++ core/Cargo.toml | 1 + core/benches/ops/main.rs | 4 +- core/benches/ops/read.rs | 7 ++-- core/benches/ops/write.rs | 2 +- core/benches/types/buffer.rs | 6 ++- core/benches/types/concurrent_tasks.rs | 4 +- core/benches/types/main.rs | 4 +- 9 files changed, 107 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/test_benchmark.yml diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml new file mode 100644 index 00000000000..347a18f40e2 --- /dev/null +++ b/.github/workflows/test_benchmark.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Benchmark Test + +on: + push: + branches: + - main + pull_request: + branches: + - main + paths: + - "core/src/**" + - "core/benches/**" + - "!core/src/docs/**" + - ".github/workflows/test_benchmark.yml" + # `workflow_dispatch` is needed by codspeed for first time trigger + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup + - name: Setup codspeed + run: cargo install cargo-codspeed + - name: Setup Memory env + uses: ./.github/services/memory/memory + - name: Build the benchmark targets + run: cargo codspeed build --features tests + - name: Run the benchmarks + uses: CodSpeedHQ/action@v3 + with: + run: cargo codspeed run + token: ${{ secrets.CODSPEED_TOKEN }} \ No newline at end of file diff --git a/core/Cargo.lock b/core/Cargo.lock index 64ba90c2a18..f0c3e7f0809 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1700,12 +1700,46 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "codspeed" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a104ac948e0188b921eb3fcbdd55dcf62e542df4c7ab7e660623f6288302089" +dependencies = [ + "colored", + "libc", + "serde_json", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "722c36bdc62d9436d027256ce2627af81ac7a596dfc7d13d849d0d212448d7fe" +dependencies = [ + "codspeed", + "colored", + "criterion", + "futures", + "tokio", +] + [[package]] name = "colorchoice" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "combine" version = "4.6.7" @@ -4794,6 +4828,7 @@ dependencies = [ "bytes", "cacache", "chrono", + "codspeed-criterion-compat", "compio", "crc32c", "criterion", diff --git a/core/Cargo.toml b/core/Cargo.toml index 50b144c5718..395f6636a47 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -380,6 +380,7 @@ probe = { version = "0.5.1", optional = true } getrandom = { version = "0.2", features = ["js"] } [dev-dependencies] +codspeed-criterion-compat = { version = "2.6.0", features = ["async", "async_tokio"] } criterion = { version = "0.5", features = ["async", "async_tokio"] } dotenvy = "0.15" fastrace = { version = "0.6", features = ["enable"] } diff --git a/core/benches/ops/main.rs b/core/benches/ops/main.rs index f79ce9515be..7d0b9ba5ccc 100644 --- a/core/benches/ops/main.rs +++ b/core/benches/ops/main.rs @@ -19,8 +19,8 @@ mod read; mod utils; mod write; -use criterion::criterion_group; -use criterion::criterion_main; +use codspeed_criterion_compat::criterion_group; +use codspeed_criterion_compat::criterion_main; criterion_group!(benches, read::bench, write::bench); criterion_main!(benches); diff --git a/core/benches/ops/read.rs b/core/benches/ops/read.rs index 95204718351..4089202fd95 100644 --- a/core/benches/ops/read.rs +++ b/core/benches/ops/read.rs @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -use criterion::Criterion; +use codspeed_criterion_compat::Criterion; +use codspeed_criterion_compat::Throughput; use futures::io; use opendal::raw::tests::init_test_service; use opendal::raw::tests::TEST_RUNTIME; @@ -47,7 +48,7 @@ fn bench_read_full(c: &mut Criterion, name: &str, op: Operator) { let path = uuid::Uuid::new_v4().to_string(); let temp_data = TempData::generate(op.clone(), &path, content.clone()); - group.throughput(criterion::Throughput::Bytes(size.bytes() as u64)); + group.throughput(Throughput::Bytes(size.bytes() as u64)); group.bench_with_input(size.to_string(), &(op.clone(), &path), |b, (op, path)| { b.to_async(&*TEST_RUNTIME).iter(|| async { let r = op.reader_with(path).await.unwrap(); @@ -84,7 +85,7 @@ fn bench_read_parallel(c: &mut Criterion, name: &str, op: Operator) { let temp_data = TempData::generate(op.clone(), &path, content.clone()); for parallel in [1, 2, 4, 8, 16] { - group.throughput(criterion::Throughput::Bytes(parallel * size.bytes() as u64)); + group.throughput(Throughput::Bytes(parallel * size.bytes() as u64)); group.bench_with_input( format!("{}x{}", parallel, size.to_string()), &(op.clone(), &path, buf_size), diff --git a/core/benches/ops/write.rs b/core/benches/ops/write.rs index 3715ac7dc29..34e29bf050d 100644 --- a/core/benches/ops/write.rs +++ b/core/benches/ops/write.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use criterion::Criterion; +use codspeed_criterion_compat::Criterion; use opendal::raw::tests::init_test_service; use opendal::raw::tests::TEST_RUNTIME; use opendal::Operator; diff --git a/core/benches/types/buffer.rs b/core/benches/types/buffer.rs index 8d9df599d6d..23a82bc9025 100644 --- a/core/benches/types/buffer.rs +++ b/core/benches/types/buffer.rs @@ -16,7 +16,9 @@ // under the License. use bytes::Buf; -use criterion::Criterion; +use codspeed_criterion_compat::Criterion; +use codspeed_criterion_compat::measurement::WallTime; +use codspeed_criterion_compat::BenchmarkGroup; use opendal::Buffer; use rand::thread_rng; use size::Size; @@ -83,7 +85,7 @@ pub fn bench_non_contiguous_buffer(c: &mut Criterion) { } pub fn bench_non_contiguous_buffer_with_extreme(c: &mut Criterion) { - let mut group: criterion::BenchmarkGroup = + let mut group: BenchmarkGroup = c.benchmark_group("bench_non_contiguous_buffer_with_extreme"); let mut rng = thread_rng(); diff --git a/core/benches/types/concurrent_tasks.rs b/core/benches/types/concurrent_tasks.rs index 3962a61f208..2fbf648d05d 100644 --- a/core/benches/types/concurrent_tasks.rs +++ b/core/benches/types/concurrent_tasks.rs @@ -17,8 +17,8 @@ use std::time::Duration; -use criterion::BatchSize; -use criterion::Criterion; +use codspeed_criterion_compat::BatchSize; +use codspeed_criterion_compat::Criterion; use once_cell::sync::Lazy; use opendal::raw::ConcurrentTasks; use opendal::Executor; diff --git a/core/benches/types/main.rs b/core/benches/types/main.rs index 801ad814f7d..ae9f74c06f3 100644 --- a/core/benches/types/main.rs +++ b/core/benches/types/main.rs @@ -19,8 +19,8 @@ mod buffer; mod concurrent_tasks; mod utils; -use criterion::criterion_group; -use criterion::criterion_main; +use codspeed_criterion_compat::criterion_group; +use codspeed_criterion_compat::criterion_main; criterion_group!( benches, From 99a90db5fa93ec6cbc30a5f4daa502ef4ca610b5 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Sat, 10 Aug 2024 18:14:07 +0800 Subject: [PATCH 02/15] chore: make cargo fmt happy --- core/benches/types/buffer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/benches/types/buffer.rs b/core/benches/types/buffer.rs index 23a82bc9025..21dfae1cb7b 100644 --- a/core/benches/types/buffer.rs +++ b/core/benches/types/buffer.rs @@ -16,9 +16,9 @@ // under the License. use bytes::Buf; -use codspeed_criterion_compat::Criterion; use codspeed_criterion_compat::measurement::WallTime; use codspeed_criterion_compat::BenchmarkGroup; +use codspeed_criterion_compat::Criterion; use opendal::Buffer; use rand::thread_rng; use size::Size; From a718e1a28540fc8999ae5c3edfd2de4214e169a9 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Tue, 22 Oct 2024 23:04:08 +0800 Subject: [PATCH 03/15] Revert "chore: make cargo fmt happy" This reverts commit 99a90db5fa93ec6cbc30a5f4daa502ef4ca610b5. --- core/benches/types/buffer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/benches/types/buffer.rs b/core/benches/types/buffer.rs index 21dfae1cb7b..23a82bc9025 100644 --- a/core/benches/types/buffer.rs +++ b/core/benches/types/buffer.rs @@ -16,9 +16,9 @@ // under the License. use bytes::Buf; +use codspeed_criterion_compat::Criterion; use codspeed_criterion_compat::measurement::WallTime; use codspeed_criterion_compat::BenchmarkGroup; -use codspeed_criterion_compat::Criterion; use opendal::Buffer; use rand::thread_rng; use size::Size; From 749254e06cbf995fa7cb0c73a3309a8cdc42c2cc Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Tue, 22 Oct 2024 23:04:09 +0800 Subject: [PATCH 04/15] Revert "chore: add codspeed for benchmark" This reverts commit c7f0c2d1759eab0b3be4facc3f31537d8ac01f98. --- .github/workflows/test_benchmark.yml | 56 -------------------------- core/Cargo.lock | 35 ---------------- core/Cargo.toml | 1 - core/benches/ops/main.rs | 4 +- core/benches/ops/read.rs | 7 ++-- core/benches/ops/write.rs | 2 +- core/benches/types/buffer.rs | 6 +-- core/benches/types/concurrent_tasks.rs | 4 +- core/benches/types/main.rs | 4 +- 9 files changed, 12 insertions(+), 107 deletions(-) delete mode 100644 .github/workflows/test_benchmark.yml diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml deleted file mode 100644 index 347a18f40e2..00000000000 --- a/.github/workflows/test_benchmark.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Benchmark Test - -on: - push: - branches: - - main - pull_request: - branches: - - main - paths: - - "core/src/**" - - "core/benches/**" - - "!core/src/docs/**" - - ".github/workflows/test_benchmark.yml" - # `workflow_dispatch` is needed by codspeed for first time trigger - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: true - -jobs: - benchmark: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - name: Setup codspeed - run: cargo install cargo-codspeed - - name: Setup Memory env - uses: ./.github/services/memory/memory - - name: Build the benchmark targets - run: cargo codspeed build --features tests - - name: Run the benchmarks - uses: CodSpeedHQ/action@v3 - with: - run: cargo codspeed run - token: ${{ secrets.CODSPEED_TOKEN }} \ No newline at end of file diff --git a/core/Cargo.lock b/core/Cargo.lock index f0c3e7f0809..64ba90c2a18 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1700,46 +1700,12 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "codspeed" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a104ac948e0188b921eb3fcbdd55dcf62e542df4c7ab7e660623f6288302089" -dependencies = [ - "colored", - "libc", - "serde_json", -] - -[[package]] -name = "codspeed-criterion-compat" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722c36bdc62d9436d027256ce2627af81ac7a596dfc7d13d849d0d212448d7fe" -dependencies = [ - "codspeed", - "colored", - "criterion", - "futures", - "tokio", -] - [[package]] name = "colorchoice" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" -[[package]] -name = "colored" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" -dependencies = [ - "lazy_static", - "windows-sys 0.48.0", -] - [[package]] name = "combine" version = "4.6.7" @@ -4828,7 +4794,6 @@ dependencies = [ "bytes", "cacache", "chrono", - "codspeed-criterion-compat", "compio", "crc32c", "criterion", diff --git a/core/Cargo.toml b/core/Cargo.toml index 395f6636a47..50b144c5718 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -380,7 +380,6 @@ probe = { version = "0.5.1", optional = true } getrandom = { version = "0.2", features = ["js"] } [dev-dependencies] -codspeed-criterion-compat = { version = "2.6.0", features = ["async", "async_tokio"] } criterion = { version = "0.5", features = ["async", "async_tokio"] } dotenvy = "0.15" fastrace = { version = "0.6", features = ["enable"] } diff --git a/core/benches/ops/main.rs b/core/benches/ops/main.rs index 7d0b9ba5ccc..f79ce9515be 100644 --- a/core/benches/ops/main.rs +++ b/core/benches/ops/main.rs @@ -19,8 +19,8 @@ mod read; mod utils; mod write; -use codspeed_criterion_compat::criterion_group; -use codspeed_criterion_compat::criterion_main; +use criterion::criterion_group; +use criterion::criterion_main; criterion_group!(benches, read::bench, write::bench); criterion_main!(benches); diff --git a/core/benches/ops/read.rs b/core/benches/ops/read.rs index 4089202fd95..95204718351 100644 --- a/core/benches/ops/read.rs +++ b/core/benches/ops/read.rs @@ -15,8 +15,7 @@ // specific language governing permissions and limitations // under the License. -use codspeed_criterion_compat::Criterion; -use codspeed_criterion_compat::Throughput; +use criterion::Criterion; use futures::io; use opendal::raw::tests::init_test_service; use opendal::raw::tests::TEST_RUNTIME; @@ -48,7 +47,7 @@ fn bench_read_full(c: &mut Criterion, name: &str, op: Operator) { let path = uuid::Uuid::new_v4().to_string(); let temp_data = TempData::generate(op.clone(), &path, content.clone()); - group.throughput(Throughput::Bytes(size.bytes() as u64)); + group.throughput(criterion::Throughput::Bytes(size.bytes() as u64)); group.bench_with_input(size.to_string(), &(op.clone(), &path), |b, (op, path)| { b.to_async(&*TEST_RUNTIME).iter(|| async { let r = op.reader_with(path).await.unwrap(); @@ -85,7 +84,7 @@ fn bench_read_parallel(c: &mut Criterion, name: &str, op: Operator) { let temp_data = TempData::generate(op.clone(), &path, content.clone()); for parallel in [1, 2, 4, 8, 16] { - group.throughput(Throughput::Bytes(parallel * size.bytes() as u64)); + group.throughput(criterion::Throughput::Bytes(parallel * size.bytes() as u64)); group.bench_with_input( format!("{}x{}", parallel, size.to_string()), &(op.clone(), &path, buf_size), diff --git a/core/benches/ops/write.rs b/core/benches/ops/write.rs index 34e29bf050d..3715ac7dc29 100644 --- a/core/benches/ops/write.rs +++ b/core/benches/ops/write.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use codspeed_criterion_compat::Criterion; +use criterion::Criterion; use opendal::raw::tests::init_test_service; use opendal::raw::tests::TEST_RUNTIME; use opendal::Operator; diff --git a/core/benches/types/buffer.rs b/core/benches/types/buffer.rs index 23a82bc9025..8d9df599d6d 100644 --- a/core/benches/types/buffer.rs +++ b/core/benches/types/buffer.rs @@ -16,9 +16,7 @@ // under the License. use bytes::Buf; -use codspeed_criterion_compat::Criterion; -use codspeed_criterion_compat::measurement::WallTime; -use codspeed_criterion_compat::BenchmarkGroup; +use criterion::Criterion; use opendal::Buffer; use rand::thread_rng; use size::Size; @@ -85,7 +83,7 @@ pub fn bench_non_contiguous_buffer(c: &mut Criterion) { } pub fn bench_non_contiguous_buffer_with_extreme(c: &mut Criterion) { - let mut group: BenchmarkGroup = + let mut group: criterion::BenchmarkGroup = c.benchmark_group("bench_non_contiguous_buffer_with_extreme"); let mut rng = thread_rng(); diff --git a/core/benches/types/concurrent_tasks.rs b/core/benches/types/concurrent_tasks.rs index 2fbf648d05d..3962a61f208 100644 --- a/core/benches/types/concurrent_tasks.rs +++ b/core/benches/types/concurrent_tasks.rs @@ -17,8 +17,8 @@ use std::time::Duration; -use codspeed_criterion_compat::BatchSize; -use codspeed_criterion_compat::Criterion; +use criterion::BatchSize; +use criterion::Criterion; use once_cell::sync::Lazy; use opendal::raw::ConcurrentTasks; use opendal::Executor; diff --git a/core/benches/types/main.rs b/core/benches/types/main.rs index ae9f74c06f3..801ad814f7d 100644 --- a/core/benches/types/main.rs +++ b/core/benches/types/main.rs @@ -19,8 +19,8 @@ mod buffer; mod concurrent_tasks; mod utils; -use codspeed_criterion_compat::criterion_group; -use codspeed_criterion_compat::criterion_main; +use criterion::criterion_group; +use criterion::criterion_main; criterion_group!( benches, From 84c474ba1c02957926dc5ea2e6cc5d158258f1b1 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Tue, 22 Oct 2024 23:11:54 +0800 Subject: [PATCH 05/15] chore: apply review suggestion to install codspeed in workflow --- .github/workflows/test_benchmark.yml | 63 ++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/test_benchmark.yml diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml new file mode 100644 index 00000000000..66969a92b79 --- /dev/null +++ b/.github/workflows/test_benchmark.yml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Benchmark Test + +on: + push: + branches: + - main + pull_request: + branches: + - main + paths: + - "core/src/**" + - "core/benches/**" + - "!core/src/docs/**" + - ".github/workflows/test_benchmark.yml" + # `workflow_dispatch` is needed by codspeed for first time trigger + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + CODSPEED_TOKEN: op://services/benchmark/codspeed + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup + - name: Setup codspeed + run: cargo install cargo-codspeed + - name: Setup Memory env + uses: ./.github/services/memory/memory + - name: Install codpseed-criterion-compat + run: cargo add --dev --rename criterion --features async,async_tokio codspeed-criterion-compat@2.7.2 + - name: Build the benchmark targets + run: cargo codspeed build --features tests + - name: Run the benchmarks + uses: CodSpeedHQ/action@v3 + with: + run: cargo codspeed run + token: ${{ env.CODSPEED_TOKEN }} \ No newline at end of file From 94818398f11926023c71dd06f56c8b44ed670deb Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Sun, 3 Nov 2024 21:04:17 +0800 Subject: [PATCH 06/15] fix(ci): add 1password setup --- .github/workflows/test_benchmark.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml index 66969a92b79..7fdccf5e7cc 100644 --- a/.github/workflows/test_benchmark.yml +++ b/.github/workflows/test_benchmark.yml @@ -40,6 +40,10 @@ jobs: benchmark: runs-on: ubuntu-latest steps: + - uses: 1password/load-secrets-action/configure@v1 + with: + connect-host: ${{ secrets.OP_CONNECT_HOST }} + connect-token: ${{ secrets.OP_CONNECT_TOKEN }} - uses: 1password/load-secrets-action@v1 with: export-env: true From 9cc403092e0edc61b3dc9122cf942068bed72fe9 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Sun, 3 Nov 2024 21:10:48 +0800 Subject: [PATCH 07/15] fix(ci): 1password setup --- .github/workflows/test_benchmark.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml index 7fdccf5e7cc..b58fb5b641b 100644 --- a/.github/workflows/test_benchmark.yml +++ b/.github/workflows/test_benchmark.yml @@ -40,6 +40,9 @@ jobs: benchmark: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup - uses: 1password/load-secrets-action/configure@v1 with: connect-host: ${{ secrets.OP_CONNECT_HOST }} @@ -49,9 +52,6 @@ jobs: export-env: true env: CODSPEED_TOKEN: op://services/benchmark/codspeed - - uses: actions/checkout@v4 - - name: Setup Rust toolchain - uses: ./.github/actions/setup - name: Setup codspeed run: cargo install cargo-codspeed - name: Setup Memory env From 5d29710f3e4b6640a0e8a75de26a9d794aa8347a Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 4 Nov 2024 18:31:01 +0800 Subject: [PATCH 08/15] Update .github/workflows/test_benchmark.yml --- .github/workflows/test_benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml index b58fb5b641b..1055c95e671 100644 --- a/.github/workflows/test_benchmark.yml +++ b/.github/workflows/test_benchmark.yml @@ -51,7 +51,7 @@ jobs: with: export-env: true env: - CODSPEED_TOKEN: op://services/benchmark/codspeed + CODSPEED_TOKEN: op://services/codspeed/token - name: Setup codspeed run: cargo install cargo-codspeed - name: Setup Memory env From 77e7fde9b66cae42d72e270fd9838f18305b5c72 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 4 Nov 2024 18:40:41 +0800 Subject: [PATCH 09/15] fix(ci): add working dir --- .github/workflows/test_benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml index 1055c95e671..01ba1c80514 100644 --- a/.github/workflows/test_benchmark.yml +++ b/.github/workflows/test_benchmark.yml @@ -59,6 +59,7 @@ jobs: - name: Install codpseed-criterion-compat run: cargo add --dev --rename criterion --features async,async_tokio codspeed-criterion-compat@2.7.2 - name: Build the benchmark targets + working-directory: core run: cargo codspeed build --features tests - name: Run the benchmarks uses: CodSpeedHQ/action@v3 From e515598770f240b5e2d77931462d603ae7f4bce7 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 4 Nov 2024 18:46:19 +0800 Subject: [PATCH 10/15] fix(ci): add working dir --- .github/workflows/test_benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml index 01ba1c80514..63fac14e678 100644 --- a/.github/workflows/test_benchmark.yml +++ b/.github/workflows/test_benchmark.yml @@ -57,6 +57,7 @@ jobs: - name: Setup Memory env uses: ./.github/services/memory/memory - name: Install codpseed-criterion-compat + working-directory: core run: cargo add --dev --rename criterion --features async,async_tokio codspeed-criterion-compat@2.7.2 - name: Build the benchmark targets working-directory: core From c7e85ebba52475a6e59a7bcd2a352cd1bf3b3def Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 4 Nov 2024 18:55:17 +0800 Subject: [PATCH 11/15] fix(ci): add working dir --- .github/workflows/test_benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml index 63fac14e678..bfbca78edf3 100644 --- a/.github/workflows/test_benchmark.yml +++ b/.github/workflows/test_benchmark.yml @@ -65,5 +65,6 @@ jobs: - name: Run the benchmarks uses: CodSpeedHQ/action@v3 with: + working-directory: core run: cargo codspeed run token: ${{ env.CODSPEED_TOKEN }} \ No newline at end of file From 37f7bca46cb8a6420e16c9a92d231caada79ea30 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 4 Nov 2024 19:45:05 +0800 Subject: [PATCH 12/15] chore(ci): skip fork repo add env --- .github/workflows/test_benchmark.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml index bfbca78edf3..967fd8e6de2 100644 --- a/.github/workflows/test_benchmark.yml +++ b/.github/workflows/test_benchmark.yml @@ -39,6 +39,7 @@ concurrency: jobs: benchmark: runs-on: ubuntu-latest + if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork steps: - uses: actions/checkout@v4 - name: Setup Rust toolchain @@ -62,6 +63,8 @@ jobs: - name: Build the benchmark targets working-directory: core run: cargo codspeed build --features tests + env: + OPENDAL_TEST: memory - name: Run the benchmarks uses: CodSpeedHQ/action@v3 with: From e26907959ad0b10d343ee1b6f2db793b092d1b0d Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 4 Nov 2024 19:45:33 +0800 Subject: [PATCH 13/15] chore(bench): make name more clear --- core/benches/types/concurrent_tasks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/benches/types/concurrent_tasks.rs b/core/benches/types/concurrent_tasks.rs index 3962a61f208..4f4274e2cb5 100644 --- a/core/benches/types/concurrent_tasks.rs +++ b/core/benches/types/concurrent_tasks.rs @@ -30,7 +30,7 @@ pub fn bench_concurrent_tasks(c: &mut Criterion) { let mut group = c.benchmark_group("bench_concurrent_tasks"); for concurrent in [1, 2, 4, 8, 16] { - group.bench_with_input(concurrent.to_string(), &concurrent, |b, concurrent| { + group.bench_with_input(format!("concurrent {}",concurrent), &concurrent, |b, concurrent| { b.to_async(&*TOKIO).iter_batched( || { ConcurrentTasks::new(Executor::new(), *concurrent, |()| { From 9f3486c957f289442edf7b50ce4448e664ac5315 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 4 Nov 2024 19:55:03 +0800 Subject: [PATCH 14/15] cargo fmt --- core/benches/types/concurrent_tasks.rs | 46 ++++++++++++++------------ 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/core/benches/types/concurrent_tasks.rs b/core/benches/types/concurrent_tasks.rs index 4f4274e2cb5..3e094854161 100644 --- a/core/benches/types/concurrent_tasks.rs +++ b/core/benches/types/concurrent_tasks.rs @@ -30,30 +30,34 @@ pub fn bench_concurrent_tasks(c: &mut Criterion) { let mut group = c.benchmark_group("bench_concurrent_tasks"); for concurrent in [1, 2, 4, 8, 16] { - group.bench_with_input(format!("concurrent {}",concurrent), &concurrent, |b, concurrent| { - b.to_async(&*TOKIO).iter_batched( - || { - ConcurrentTasks::new(Executor::new(), *concurrent, |()| { - Box::pin(async { - tokio::time::sleep(Duration::from_millis(1)).await; - ((), Ok(())) + group.bench_with_input( + format!("concurrent {}", concurrent), + &concurrent, + |b, concurrent| { + b.to_async(&*TOKIO).iter_batched( + || { + ConcurrentTasks::new(Executor::new(), *concurrent, |()| { + Box::pin(async { + tokio::time::sleep(Duration::from_millis(1)).await; + ((), Ok(())) + }) }) - }) - }, - |mut tasks| async move { - for _ in 0..100 { - let _ = tasks.execute(()).await; - } + }, + |mut tasks| async move { + for _ in 0..100 { + let _ = tasks.execute(()).await; + } - loop { - if tasks.next().await.is_none() { - break; + loop { + if tasks.next().await.is_none() { + break; + } } - } - }, - BatchSize::PerIteration, - ) - }); + }, + BatchSize::PerIteration, + ) + }, + ); } group.finish() From 4e0ff7e67ee4b12be6af44d4715d6ba3950667a8 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 4 Nov 2024 21:53:56 +0800 Subject: [PATCH 15/15] Update .github/workflows/test_benchmark.yml --- .github/workflows/test_benchmark.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_benchmark.yml b/.github/workflows/test_benchmark.yml index 967fd8e6de2..ccaa30724b2 100644 --- a/.github/workflows/test_benchmark.yml +++ b/.github/workflows/test_benchmark.yml @@ -62,11 +62,11 @@ jobs: run: cargo add --dev --rename criterion --features async,async_tokio codspeed-criterion-compat@2.7.2 - name: Build the benchmark targets working-directory: core - run: cargo codspeed build --features tests - env: - OPENDAL_TEST: memory + run: cargo codspeed build --features tests,services-memory - name: Run the benchmarks uses: CodSpeedHQ/action@v3 + env: + OPENDAL_TEST: memory with: working-directory: core run: cargo codspeed run