diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c3ba6487e..25c2dc68c 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -8,22 +8,22 @@ Latest commit hash in PR branch will be built and used for benchmarking. ### How to execute Usage ``` -# [chain names] - Use "," for multiple runtimes. Available values are: astar-dev, shiden-dev, shibuya-dev, dev +# [chain names] - Use "," for multiple runtimes. Available values are: astar, shiden, shibuya # [pallet names] - Use "," for multiple pallets, "all" for all pallets /bench [chain names] [pallet names] ``` ``` # benchmark a pallet -/bench astar-dev pallet_balances +/bench astar pallet_balances # benchmark multiple pallets -/bench astar-dev pallet_balances,pallet_dapps_staking +/bench astar pallet_balances,pallet_dapps_staking # benchmark all pallets -/bench astar-dev all +/bench astar all # benchmark multiple runtimes with multiple pallets -/bench astar-dev,shibuya-dev pallet_balances,pallet_dapps_staking +/bench astar,shibuya pallet_balances,pallet_dapps_staking ``` diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4fe6ea1eb..a9984e785 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -109,6 +109,9 @@ jobs: - name: Check targets are installed correctly run: rustup target list --installed + - name: Install frame-omni bencher tool + run: cargo install frame-omni-bencher + - name: Execute benchmarking run: | mkdir -p ./benchmark-results diff --git a/scripts/run_benchmarks.sh b/scripts/run_benchmarks.sh index c739e4762..4b51f2a9d 100755 --- a/scripts/run_benchmarks.sh +++ b/scripts/run_benchmarks.sh @@ -30,7 +30,7 @@ while getopts 'bc:fo:p:v' flag; do ;; c) chains=$(echo ${OPTARG} | tr '[:upper:]' '[:lower:]') - chains_default=("astar-dev" "shiden-dev" "shibuya-dev" "dev") + chains_default=("astar" "shiden" "shibuya") for chain in ${chains//,/ }; do if [[ ! " ${chains_default[*]} " =~ " ${chain} " ]]; then echo "Chain input is invalid. ${chain} not included in ${chains_default[*]}" @@ -73,11 +73,12 @@ done if [ "$skip_build" != true ] then echo "[+] Compiling astar-collator benchmarks..." - CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" cargo build --release --verbose --features=runtime-benchmarks + CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" cargo build --release --verbose --features=runtime-benchmarks \ + -p astar-runtime -p shiden-runtime -p shibuya-runtime fi # The executable to use. -ASTAR_COLLATOR=./target/release/astar-collator +BENCHMARK_TOOL="frame-omni-bencher v1" # Manually exclude some pallets. EXCLUDED_PALLETS=( @@ -86,7 +87,7 @@ EXCLUDED_PALLETS=( # Load all pallet names in an array. ALL_PALLETS=($( - $ASTAR_COLLATOR benchmark pallet --list --chain=$chain |\ + $BENCHMARK_TOOL benchmark pallet --list --runtime ./target/release/wbuild/${chain}-runtime/${chain}_runtime.compact.compressed.wasm |\ tail -n+2 |\ cut -d',' -f1 |\ sort |\ @@ -120,8 +121,8 @@ for chain in ${chains//,/ }; do echo "[+] Benchmarking $PALLET with weight file $WEIGHT_FILE"; OUTPUT=$( - $ASTAR_COLLATOR benchmark pallet \ - --chain=$chain \ + $BENCHMARK_TOOL benchmark pallet \ + --runtime ./target/release/wbuild/${chain}-runtime/${chain}_runtime.compact.compressed.wasm \ --steps=50 \ --repeat=20 \ --pallet="$PALLET" \ @@ -137,16 +138,19 @@ for chain in ${chains//,/ }; do fi done - echo "[+] Benchmarking the machine..." - OUTPUT=$( - $ASTAR_COLLATOR benchmark machine --chain=$chain 2>&1 - ) - if [ $? -ne 0 ]; then - echo "[-] Failed the machine benchmark" - echo "$OUTPUT" >> "$ERR_FILE" - else - echo "$OUTPUT" >> "$output_path/$chain/$chain-machine-bench.txt" - fi + # Disabled for now - command isn't available (yet). + # With latest changes we also benchmark the HW each time the client starts. + # + # echo "[+] Benchmarking the machine..." + # OUTPUT=$( + # $BENCHMARK_TOOL benchmark machine --runtime ./target/release/wbuild/astar-runtime/${chain}_runtime.compact.compressed.wasm 2>&1 + # ) + # if [ $? -ne 0 ]; then + # echo "[-] Failed the machine benchmark" + # echo "$OUTPUT" >> "$ERR_FILE" + # else + # echo "$OUTPUT" >> "$output_path/$chain/$chain-machine-bench.txt" + # fi # Check if the error file exists. if [ -f "$ERR_FILE" ]; then