From 11c1538fd4573f4dd8752a084f7561df54be2f4e Mon Sep 17 00:00:00 2001
From: Javier Bullrich <javier@bullrich.dev>
Date: Mon, 10 Jun 2024 14:26:51 +0200
Subject: [PATCH] added lib scripts

---
 scripts/lib/bench-all-cumulus.sh   | 139 ++++++++++++++++++++++
 scripts/lib/bench-all-polkadot.sh  |  88 ++++++++++++++
 scripts/lib/bench-all-substrate.sh | 148 ++++++++++++++++++++++++
 scripts/lib/bench-overhead.sh      |  66 +++++++++++
 scripts/lib/bench-pallet.sh        | 178 +++++++++++++++++++++++++++++
 5 files changed, 619 insertions(+)
 create mode 100755 scripts/lib/bench-all-cumulus.sh
 create mode 100644 scripts/lib/bench-all-polkadot.sh
 create mode 100644 scripts/lib/bench-all-substrate.sh
 create mode 100644 scripts/lib/bench-overhead.sh
 create mode 100644 scripts/lib/bench-pallet.sh

diff --git a/scripts/lib/bench-all-cumulus.sh b/scripts/lib/bench-all-cumulus.sh
new file mode 100755
index 0000000000000..f4c2a35c6b6b7
--- /dev/null
+++ b/scripts/lib/bench-all-cumulus.sh
@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+# originally moved from https://github.com/paritytech/cumulus/blob/445f9277ab55b4d930ced4fbbb38d27c617c6658/scripts/benchmarks-ci.sh
+
+# default RUST_LOG is warn, but could be overridden
+export RUST_LOG="${RUST_LOG:-error}"
+
+THIS_DIR=$(dirname "${BASH_SOURCE[0]}")
+. "$THIS_DIR/../command-utils.sh"
+
+POLKADOT_PARACHAIN="./target/$profile/polkadot-parachain"
+
+run_cumulus_bench() {
+  local artifactsDir="$ARTIFACTS_DIR"
+  local category=$1
+  local runtimeName=$2
+  local paraId=${3:-}
+
+  local benchmarkOutput="$output_path/parachains/runtimes/$category/$runtimeName/src/weights"
+  local benchmarkRuntimeChain
+  if [[ ! -z "$paraId" ]]; then
+     benchmarkRuntimeChain="${runtimeName}-dev-$paraId"
+  else
+     benchmarkRuntimeChain="$runtimeName-dev"
+  fi
+
+  local benchmarkMetadataOutputDir="$artifactsDir/$runtimeName"
+  mkdir -p "$benchmarkMetadataOutputDir"
+
+  # Load all pallet names in an array.
+  echo "[+] Listing pallets for runtime $runtimeName for chain: $benchmarkRuntimeChain ..."
+  local pallets=($(
+    $POLKADOT_PARACHAIN benchmark pallet --list --chain="${benchmarkRuntimeChain}" |\
+      tail -n+2 |\
+      cut -d',' -f1 |\
+      sort |\
+      uniq
+  ))
+
+  if [ ${#pallets[@]} -ne 0 ]; then
+    echo "[+] Benchmarking ${#pallets[@]} pallets for runtime $runtimeName for chain: $benchmarkRuntimeChain, pallets:"
+    for pallet in "${pallets[@]}"; do
+        echo "   [+] $pallet"
+    done
+  else
+    echo "$runtimeName pallet list not found in benchmarks-ci.sh"
+    exit 1
+  fi
+
+  for pallet in "${pallets[@]}"; do
+    # (by default) do not choose output_file, like `pallet_assets.rs` because it does not work for multiple instances
+    # `benchmark pallet` command will decide the output_file name if there are multiple instances
+    local output_file=""
+    local extra_args=""
+    # a little hack for pallet_xcm_benchmarks - we want to force custom implementation for XcmWeightInfo
+    if [[ "$pallet" == "pallet_xcm_benchmarks::generic" ]] || [[ "$pallet" == "pallet_xcm_benchmarks::fungible" ]]; then
+      output_file="xcm/${pallet//::/_}.rs"
+      extra_args="--template=$output_path/templates/xcm-bench-template.hbs"
+    fi
+    $POLKADOT_PARACHAIN benchmark pallet \
+      $extra_args \
+      --chain="${benchmarkRuntimeChain}" \
+      --wasm-execution=compiled \
+      --pallet="$pallet" \
+      --no-storage-info \
+      --no-median-slopes \
+      --no-min-squares \
+      --extrinsic='*' \
+      --steps=50 \
+      --repeat=20 \
+      --json \
+      --header="$output_path/file_header.txt" \
+      --output="${benchmarkOutput}/${output_file}" >> "$benchmarkMetadataOutputDir/${pallet//::/_}_benchmark.json"
+  done
+}
+
+
+echo "[+] Compiling benchmarks..."
+cargo build --profile $profile --locked --features=runtime-benchmarks -p polkadot-parachain-bin
+
+# Run benchmarks for all pallets of a given runtime if runtime argument provided
+get_arg optional --runtime "$@"
+runtime="${out:-""}"
+
+if [[ $runtime ]]; then
+  paraId=""
+  case "$runtime" in
+    asset-*)
+      category="assets"
+    ;;
+    collectives-*)
+      category="collectives"
+    ;;
+    coretime-*)
+      category="coretime"
+    ;;
+    bridge-*)
+      category="bridge-hubs"
+    ;;
+    contracts-*)
+      category="contracts"
+    ;;
+    people-*)
+      category="people"
+    ;;
+    glutton-*)
+      category="glutton"
+      paraId="1300"
+    ;;
+    *)
+      echo "Unknown runtime: $runtime"
+      exit 1
+    ;;
+  esac
+
+  run_cumulus_bench $category $runtime $paraId
+
+else # run all
+  # Assets
+  run_cumulus_bench assets asset-hub-rococo
+  run_cumulus_bench assets asset-hub-westend
+
+  # Collectives
+  run_cumulus_bench collectives collectives-westend
+
+  # Coretime
+  run_cumulus_bench coretime coretime-rococo
+  run_cumulus_bench coretime coretime-westend
+
+  # People
+  run_cumulus_bench people people-rococo
+  run_cumulus_bench people people-westend
+
+  # Bridge Hubs
+  run_cumulus_bench bridge-hubs bridge-hub-rococo
+  run_cumulus_bench bridge-hubs bridge-hub-westend
+
+  # Glutton
+  run_cumulus_bench glutton glutton-westend 1300
+fi
diff --git a/scripts/lib/bench-all-polkadot.sh b/scripts/lib/bench-all-polkadot.sh
new file mode 100644
index 0000000000000..ac52e00140e38
--- /dev/null
+++ b/scripts/lib/bench-all-polkadot.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# Runs all benchmarks for all pallets, for a given runtime, provided by $1
+# Should be run on a reference machine to gain accurate benchmarks
+# current reference machine: https://github.com/paritytech/polkadot/pull/6508/files
+# original source: https://github.com/paritytech/polkadot/blob/b9842c4b52f6791fef6c11ecd020b22fe614f041/scripts/run_all_benches.sh
+
+get_arg required --runtime "$@"
+runtime="${out:-""}"
+
+# default RUST_LOG is error, but could be overridden
+export RUST_LOG="${RUST_LOG:-error}"
+
+echo "[+] Compiling benchmarks..."
+cargo build --profile $profile --locked --features=runtime-benchmarks -p polkadot
+
+POLKADOT_BIN="./target/$profile/polkadot"
+
+# Update the block and extrinsic overhead weights.
+echo "[+] Benchmarking block and extrinsic overheads..."
+OUTPUT=$(
+  $POLKADOT_BIN benchmark overhead \
+  --chain="${runtime}-dev" \
+  --wasm-execution=compiled \
+  --weight-path="$output_path/runtime/${runtime}/constants/src/weights/" \
+  --warmup=10 \
+  --repeat=100 \
+  --header="$output_path/file_header.txt"
+)
+if [ $? -ne 0 ]; then
+  echo "$OUTPUT" >> "$ERR_FILE"
+  echo "[-] Failed to benchmark the block and extrinsic overheads. Error written to $ERR_FILE; continuing..."
+fi
+
+
+# Load all pallet names in an array.
+PALLETS=($(
+  $POLKADOT_BIN benchmark pallet --list --chain="${runtime}-dev" |\
+    tail -n+2 |\
+    cut -d',' -f1 |\
+    sort |\
+    uniq
+))
+
+echo "[+] Benchmarking ${#PALLETS[@]} pallets for runtime $runtime"
+
+# Define the error file.
+ERR_FILE="${ARTIFACTS_DIR}/benchmarking_errors.txt"
+# Delete the error file before each run.
+rm -f $ERR_FILE
+
+# Benchmark each pallet.
+for PALLET in "${PALLETS[@]}"; do
+  echo "[+] Benchmarking $PALLET for $runtime";
+
+  output_file=""
+  if [[ $PALLET == *"::"* ]]; then
+    # translates e.g. "pallet_foo::bar" to "pallet_foo_bar"
+    output_file="${PALLET//::/_}.rs"
+  fi
+
+  OUTPUT=$(
+    $POLKADOT_BIN benchmark pallet \
+    --chain="${runtime}-dev" \
+    --steps=50 \
+    --repeat=20 \
+    --no-storage-info \
+    --no-median-slopes \
+    --no-min-squares \
+    --pallet="$PALLET" \
+    --extrinsic="*" \
+    --execution=wasm \
+    --wasm-execution=compiled \
+    --header="$output_path/file_header.txt" \
+    --output="$output_path/runtime/${runtime}/src/weights/${output_file}" 2>&1
+  )
+  if [ $? -ne 0 ]; then
+    echo "$OUTPUT" >> "$ERR_FILE"
+    echo "[-] Failed to benchmark $PALLET. Error written to $ERR_FILE; continuing..."
+  fi
+done
+
+# Check if the error file exists.
+if [ -f "$ERR_FILE" ]; then
+  echo "[-] Some benchmarks failed. See: $ERR_FILE"
+else
+  echo "[+] All benchmarks passed."
+fi
diff --git a/scripts/lib/bench-all-substrate.sh b/scripts/lib/bench-all-substrate.sh
new file mode 100644
index 0000000000000..eeb18cdd8bbb3
--- /dev/null
+++ b/scripts/lib/bench-all-substrate.sh
@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+# This file is part of Substrate.
+# Copyright (C) 2022 Parity Technologies (UK) Ltd.
+# SPDX-License-Identifier: Apache-2.0
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script has three parts which all use the Substrate runtime:
+# - Pallet benchmarking to update the pallet weights
+# - Overhead benchmarking for the Extrinsic and Block weights
+# - Machine benchmarking
+#
+# Should be run on a reference machine to gain accurate benchmarks
+# current reference machine: https://github.com/paritytech/substrate/pull/5848
+
+# Original source: https://github.com/paritytech/substrate/blob/ff9921a260a67e3a71f25c8b402cd5c7da787a96/scripts/run_all_benchmarks.sh
+# Fail if any sub-command in a pipe fails, not just the last one.
+set -o pipefail
+# Fail on undeclared variables.
+set -u
+# Fail if any sub-command fails.
+set -e
+# Fail on traps.
+# set -E
+
+# default RUST_LOG is warn, but could be overridden
+export RUST_LOG="${RUST_LOG:-error}"
+
+echo "[+] Compiling Substrate benchmarks..."
+cargo build --profile=$profile --locked --features=runtime-benchmarks -p staging-node-cli
+
+# The executable to use.
+SUBSTRATE="./target/$profile/substrate-node"
+
+# Manually exclude some pallets.
+EXCLUDED_PALLETS=(
+  # Helper pallets
+  "pallet_election_provider_support_benchmarking"
+  # Pallets without automatic benchmarking
+  "pallet_babe"
+  "pallet_grandpa"
+  "pallet_mmr"
+  "pallet_offences"
+  # Only used for testing, does not need real weights.
+  "frame_benchmarking_pallet_pov"
+  "pallet_example_tasks"
+  "pallet_example_basic"
+  "pallet_example_split"
+  "pallet_example_kitchensink"
+  "pallet_example_mbm"
+  "tasks_example"
+)
+
+# Load all pallet names in an array.
+ALL_PALLETS=($(
+  $SUBSTRATE benchmark pallet --list --chain=dev |\
+    tail -n+2 |\
+    cut -d',' -f1 |\
+    sort |\
+    uniq
+))
+
+# Define the error file.
+ERR_FILE="${ARTIFACTS_DIR}/benchmarking_errors.txt"
+
+# Delete the error file before each run.
+rm -f "$ERR_FILE"
+
+mkdir -p "$(dirname "$ERR_FILE")"
+
+# Update the block and extrinsic overhead weights.
+echo "[+] Benchmarking block and extrinsic overheads..."
+OUTPUT=$(
+  $SUBSTRATE benchmark overhead \
+  --chain=dev \
+  --wasm-execution=compiled \
+  --weight-path="$output_path/frame/support/src/weights/" \
+  --header="$output_path/HEADER-APACHE2" \
+  --warmup=10 \
+  --repeat=100 2>&1
+)
+if [ $? -ne 0 ]; then
+  echo "$OUTPUT" >> "$ERR_FILE"
+  echo "[-] Failed to benchmark the block and extrinsic overheads. Error written to $ERR_FILE; continuing..."
+fi
+
+echo "[+] Benchmarking ${#ALL_PALLETS[@]} Substrate pallets and excluding ${#EXCLUDED_PALLETS[@]}."
+
+echo "[+] Excluded pallets ${EXCLUDED_PALLETS[@]}"
+echo "[+] ------ "
+echo "[+] Whole list pallets ${ALL_PALLETS[@]}"
+
+# Benchmark each pallet.
+for PALLET in "${ALL_PALLETS[@]}"; do
+  FOLDER="$(echo "${PALLET#*_}" | tr '_' '-')";
+  WEIGHT_FILE="$output_path/frame/${FOLDER}/src/weights.rs"
+
+   # Skip the pallet if it is in the excluded list.
+
+  if [[ " ${EXCLUDED_PALLETS[@]} " =~ " ${PALLET} " ]]; then
+    echo "[+] Skipping $PALLET as it is in the excluded list."
+    continue
+  fi
+
+  echo "[+] Benchmarking $PALLET with weight file $WEIGHT_FILE";
+
+  set +e # Disable exit on error for the benchmarking of the pallets
+  OUTPUT=$(
+    $SUBSTRATE benchmark pallet \
+    --chain=dev \
+    --steps=50 \
+    --repeat=20 \
+    --pallet="$PALLET" \
+    --no-storage-info \
+    --no-median-slopes \
+    --no-min-squares \
+    --extrinsic="*" \
+    --wasm-execution=compiled \
+    --heap-pages=4096 \
+    --output="$WEIGHT_FILE" \
+    --header="$output_path/HEADER-APACHE2" \
+    --template="$output_path/.maintain/frame-weight-template.hbs" 2>&1
+  )
+  if [ $? -ne 0 ]; then
+    echo -e "$PALLET: $OUTPUT\n" >> "$ERR_FILE"
+    echo "[-] Failed to benchmark $PALLET. Error written to $ERR_FILE; continuing..."
+  fi
+  set -e # Re-enable exit on error
+done
+
+
+# Check if the error file exists.
+if [ -s "$ERR_FILE" ]; then
+  echo "[-] Some benchmarks failed. See: $ERR_FILE"
+  exit 1
+else
+  echo "[+] All benchmarks passed."
+fi
diff --git a/scripts/lib/bench-overhead.sh b/scripts/lib/bench-overhead.sh
new file mode 100644
index 0000000000000..c4cca8b4c128c
--- /dev/null
+++ b/scripts/lib/bench-overhead.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+THIS_DIR=$(dirname "${BASH_SOURCE[0]}")
+. "$THIS_DIR/../command-utils.sh"
+
+bench_overhead_common_args=(
+  --
+  benchmark
+  overhead
+  --wasm-execution=compiled
+  --warmup=10
+  --repeat=100
+)
+bench_overhead() {
+  local args
+  case "$target_dir" in
+    substrate)
+      args=(
+        --bin=substrate
+        "${bench_overhead_common_args[@]}"
+        --header="$output_path/HEADER-APACHE2"
+        --weight-path="$output_path/frame/support/src/weights"
+        --chain="dev"
+      )
+    ;;
+    polkadot)
+      get_arg required --runtime "$@"
+      local runtime="${out:-""}"
+      args=(
+        --bin=polkadot
+        "${bench_overhead_common_args[@]}"
+        --header="$output_path/file_header.txt"
+        --weight-path="$output_path/runtime/$runtime/constants/src/weights"
+        --chain="$runtime-dev"
+      )
+    ;;
+    cumulus)
+      get_arg required --runtime "$@"
+      local runtime="${out:-""}"
+      args=(
+        -p=polkadot-parachain-bin
+        "${bench_overhead_common_args[@]}"
+        --header="$output_path/file_header.txt"
+        --weight-path="$output_path/parachains/runtimes/assets/$runtime/src/weights"
+        --chain="$runtime"
+      )
+    ;;
+    trappist)
+      get_arg required --runtime "$@"
+      local runtime="${out:-""}"
+      args=(
+        "${bench_overhead_common_args[@]}"
+        --header="$output_path/templates/file_header.txt"
+        --weight-path="$output_path/runtime/$runtime/src/weights"
+        --chain="$runtime-dev"
+      )
+    ;;
+    *)
+      die "Target Dir \"$target_dir\" is not supported in bench_overhead"
+    ;;
+  esac
+
+  cargo_run "${args[@]}"
+}
+
+bench_overhead "$@"
diff --git a/scripts/lib/bench-pallet.sh b/scripts/lib/bench-pallet.sh
new file mode 100644
index 0000000000000..15eac31e3a45c
--- /dev/null
+++ b/scripts/lib/bench-pallet.sh
@@ -0,0 +1,178 @@
+#!/bin/bash
+
+THIS_DIR=$(dirname "${BASH_SOURCE[0]}")
+. "$THIS_DIR/../command-utils.sh"
+
+bench_pallet_common_args=(
+  --
+  benchmark
+  pallet
+  --steps=50
+  --repeat=20
+  --extrinsic="*"
+  --wasm-execution=compiled
+  --heap-pages=4096
+  --json-file="${ARTIFACTS_DIR}/bench.json"
+)
+bench_pallet() {
+  get_arg required --subcommand "$@"
+  local subcommand="${out:-""}"
+
+  get_arg required --runtime "$@"
+  local runtime="${out:-""}"
+
+  get_arg required --pallet "$@"
+  local pallet="${out:-""}"
+
+  local args
+  case "$target_dir" in
+    substrate)
+      args=(
+        --features=runtime-benchmarks
+        --manifest-path="$output_path/bin/node/cli/Cargo.toml"
+        "${bench_pallet_common_args[@]}"
+        --pallet="$pallet"
+        --chain="$runtime"
+      )
+
+      case "$subcommand" in
+        pallet)
+          # Translates e.g. "pallet_foo::bar" to "pallet_foo_bar"
+          local output_dir="${pallet//::/_}"
+
+          # Substrate benchmarks are output to the "frame" directory but they aren't
+          # named exactly after the $pallet argument. For example:
+          # - When $pallet == pallet_balances, the output folder is frame/balances
+          # - When $pallet == frame_benchmarking, the output folder is frame/benchmarking
+          # The common pattern we infer from those examples is that we should remove
+          # the prefix
+          if [[ "$output_dir" =~ ^[A-Za-z]*[^A-Za-z](.*)$ ]]; then
+            output_dir="${BASH_REMATCH[1]}"
+          fi
+
+          # We also need to translate '_' to '-' due to the folders' naming
+          # conventions
+          output_dir="${output_dir//_/-}"
+
+          args+=(
+            --header="$output_path/HEADER-APACHE2"
+            --output="$output_path/frame/$output_dir/src/weights.rs"
+            --template="$output_path/.maintain/frame-weight-template.hbs"
+          )
+        ;;
+        *)
+          die "Subcommand $subcommand is not supported for $target_dir in bench_pallet"
+        ;;
+      esac
+    ;;
+    polkadot)
+      # For backward compatibility: replace "-dev" with ""
+      runtime=${runtime/-dev/}
+
+      local weights_dir="$output_path/runtime/${runtime}/src/weights"
+
+      args=(
+        --bin=polkadot
+        --features=runtime-benchmarks
+        "${bench_pallet_common_args[@]}"
+        --pallet="$pallet"
+        --chain="${runtime}-dev"
+      )
+
+      case "$subcommand" in
+        pallet)
+          args+=(
+            --header="$output_path/file_header.txt"
+            --output="${weights_dir}/"
+          )
+        ;;
+        xcm)
+          args+=(
+            --header="$output_path/file_header.txt"
+            --template="$output_path/xcm/pallet-xcm-benchmarks/template.hbs"
+            --output="${weights_dir}/xcm/"
+          )
+        ;;
+        *)
+          die "Subcommand $subcommand is not supported for $target_dir in bench_pallet"
+        ;;
+      esac
+    ;;
+    cumulus)
+      get_arg required --runtime_dir "$@"
+      local runtime_dir="${out:-""}"
+      local chain="$runtime"
+
+      # to support specifying parachain id from runtime name (e.g. ["glutton-westend", "glutton-westend-dev-1300"])
+      # If runtime ends with "-dev" or "-dev-\d+", leave as it is, otherwise concat "-dev" at the end of $chain
+      if [[ ! "$runtime" =~ -dev(-[0-9]+)?$ ]]; then
+          chain="${runtime}-dev"
+      fi
+
+      # replace "-dev" or "-dev-\d+" with "" for runtime
+      runtime=$(echo "$runtime" | sed 's/-dev.*//g')
+
+      args=(
+        -p=polkadot-parachain-bin
+        --features=runtime-benchmarks
+        "${bench_pallet_common_args[@]}"
+        --pallet="$pallet"
+        --chain="${chain}"
+        --header="$output_path/file_header.txt"
+      )
+
+      case "$subcommand" in
+        pallet)
+          args+=(
+            --output="$output_path/parachains/runtimes/$runtime_dir/$runtime/src/weights/"
+          )
+        ;;
+        xcm)
+          mkdir -p "$output_path/parachains/runtimes/$runtime_dir/$runtime/src/weights/xcm"
+          args+=(
+            --template="$output_path/templates/xcm-bench-template.hbs"
+            --output="$output_path/parachains/runtimes/$runtime_dir/$runtime/src/weights/xcm/"
+          )
+        ;;
+        *)
+          die "Subcommand $subcommand is not supported for $target_dir in bench_pallet"
+        ;;
+      esac
+    ;;
+    trappist)
+      local weights_dir="$output_path/runtime/$runtime/src/weights"
+
+      args=(
+        --features=runtime-benchmarks
+        "${bench_pallet_common_args[@]}"
+        --pallet="$pallet"
+        --chain="${runtime}-dev"
+        --header="$output_path/templates/file_header.txt"
+      )
+
+      case "$subcommand" in
+        pallet)
+          args+=(
+            --output="${weights_dir}/"
+          )
+        ;;
+        xcm)
+          args+=(
+            --template="$output_path/templates/xcm-bench-template.hbs"
+            --output="${weights_dir}/xcm/"
+          )
+        ;;
+        *)
+          die "Subcommand $subcommand is not supported for $target_dir in bench_pallet"
+        ;;
+      esac
+    ;;
+    *)
+      die "Repository $target_dir is not supported in bench_pallet"
+    ;;
+  esac
+
+  cargo_run "${args[@]}"
+}
+
+bench_pallet "$@"