Merge branch 'master' into cmd-test

scripts/lib/bench-all-cumulus.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+# originally moved from https://github.com/paritytech/cumulus/blob/445f9277ab55b4d930ced4fbbb38d27c617c6658/scripts/benchmarks-ci.sh
+
+# default RUST_LOG is warn, but could be overridden
+export RUST_LOG="${RUST_LOG:-error}"
+
+THIS_DIR=$(dirname "${BASH_SOURCE[0]}")
+. "$THIS_DIR/../command-utils.sh"
+
+POLKADOT_PARACHAIN="./target/$profile/polkadot-parachain"
+
+run_cumulus_bench() {
+  local artifactsDir="$ARTIFACTS_DIR"
+  local category=$1
+  local runtimeName=$2
+  local paraId=${3:-}
+
+  local benchmarkOutput="$output_path/parachains/runtimes/$category/$runtimeName/src/weights"
+  local benchmarkRuntimeChain
+  if [[ ! -z "$paraId" ]]; then
+     benchmarkRuntimeChain="${runtimeName}-dev-$paraId"
+  else
+     benchmarkRuntimeChain="$runtimeName-dev"
+  fi
+
+  local benchmarkMetadataOutputDir="$artifactsDir/$runtimeName"
+  mkdir -p "$benchmarkMetadataOutputDir"
+
+  # Load all pallet names in an array.
+  echo "[+] Listing pallets for runtime $runtimeName for chain: $benchmarkRuntimeChain ..."
+  local pallets=($(
+    $POLKADOT_PARACHAIN benchmark pallet --list --chain="${benchmarkRuntimeChain}" |\
+      tail -n+2 |\
+      cut -d',' -f1 |\
+      sort |\
+      uniq
+  ))
+
+  if [ ${#pallets[@]} -ne 0 ]; then
+    echo "[+] Benchmarking ${#pallets[@]} pallets for runtime $runtimeName for chain: $benchmarkRuntimeChain, pallets:"
+    for pallet in "${pallets[@]}"; do
+        echo "   [+] $pallet"
+    done
+  else
+    echo "$runtimeName pallet list not found in benchmarks-ci.sh"
+    exit 1
+  fi
+
+  for pallet in "${pallets[@]}"; do
+    # (by default) do not choose output_file, like `pallet_assets.rs` because it does not work for multiple instances
+    # `benchmark pallet` command will decide the output_file name if there are multiple instances
+    local output_file=""
+    local extra_args=""
+    # a little hack for pallet_xcm_benchmarks - we want to force custom implementation for XcmWeightInfo
+    if [[ "$pallet" == "pallet_xcm_benchmarks::generic" ]] || [[ "$pallet" == "pallet_xcm_benchmarks::fungible" ]]; then
+      output_file="xcm/${pallet//::/_}.rs"
+      extra_args="--template=$output_path/templates/xcm-bench-template.hbs"
+    fi
+    $POLKADOT_PARACHAIN benchmark pallet \
+      $extra_args \
+      --chain="${benchmarkRuntimeChain}" \
+      --wasm-execution=compiled \
+      --pallet="$pallet" \
+      --no-storage-info \
+      --no-median-slopes \
+      --no-min-squares \
+      --extrinsic='*' \
+      --steps=50 \
+      --repeat=20 \
+      --json \
+      --header="$output_path/file_header.txt" \
+      --output="${benchmarkOutput}/${output_file}" >> "$benchmarkMetadataOutputDir/${pallet//::/_}_benchmark.json"
+  done
+}
+
+
+echo "[+] Compiling benchmarks..."
+cargo build --profile $profile --locked --features=runtime-benchmarks -p polkadot-parachain-bin
+
+# Run benchmarks for all pallets of a given runtime if runtime argument provided
+get_arg optional --runtime "$@"
+runtime="${out:-""}"
+
+if [[ $runtime ]]; then
+  paraId=""
+  case "$runtime" in
+    asset-*)
+      category="assets"
+    ;;
+    collectives-*)
+      category="collectives"
+    ;;
+    coretime-*)
+      category="coretime"
+    ;;
+    bridge-*)
+      category="bridge-hubs"
+    ;;
+    contracts-*)
+      category="contracts"
+    ;;
+    people-*)
+      category="people"
+    ;;
+    glutton-*)
+      category="glutton"
+      paraId="1300"
+    ;;
+    *)
+      echo "Unknown runtime: $runtime"
+      exit 1
+    ;;
+  esac
+
+  run_cumulus_bench $category $runtime $paraId
+
+else # run all
+  # Assets
+  run_cumulus_bench assets asset-hub-rococo
+  run_cumulus_bench assets asset-hub-westend
+
+  # Collectives
+  run_cumulus_bench collectives collectives-westend
+
+  # Coretime
+  run_cumulus_bench coretime coretime-rococo
+  run_cumulus_bench coretime coretime-westend
+
+  # People
+  run_cumulus_bench people people-rococo
+  run_cumulus_bench people people-westend
+
+  # Bridge Hubs
+  run_cumulus_bench bridge-hubs bridge-hub-rococo
+  run_cumulus_bench bridge-hubs bridge-hub-westend
+
+  # Glutton
+  run_cumulus_bench glutton glutton-westend 1300
+fi

scripts/lib/bench-all-polkadot.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# Runs all benchmarks for all pallets, for a given runtime, provided by $1
+# Should be run on a reference machine to gain accurate benchmarks
+# current reference machine: https://github.com/paritytech/polkadot/pull/6508/files
+# original source: https://github.com/paritytech/polkadot/blob/b9842c4b52f6791fef6c11ecd020b22fe614f041/scripts/run_all_benches.sh
+
+get_arg required --runtime "$@"
+runtime="${out:-""}"
+
+# default RUST_LOG is error, but could be overridden
+export RUST_LOG="${RUST_LOG:-error}"
+
+echo "[+] Compiling benchmarks..."
+cargo build --profile $profile --locked --features=runtime-benchmarks -p polkadot
+
+POLKADOT_BIN="./target/$profile/polkadot"
+
+# Update the block and extrinsic overhead weights.
+echo "[+] Benchmarking block and extrinsic overheads..."
+OUTPUT=$(
+  $POLKADOT_BIN benchmark overhead \
+  --chain="${runtime}-dev" \
+  --wasm-execution=compiled \
+  --weight-path="$output_path/runtime/${runtime}/constants/src/weights/" \
+  --warmup=10 \
+  --repeat=100 \
+  --header="$output_path/file_header.txt"
+)
+if [ $? -ne 0 ]; then
+  echo "$OUTPUT" >> "$ERR_FILE"
+  echo "[-] Failed to benchmark the block and extrinsic overheads. Error written to $ERR_FILE; continuing..."
+fi
+
+
+# Load all pallet names in an array.
+PALLETS=($(
+  $POLKADOT_BIN benchmark pallet --list --chain="${runtime}-dev" |\
+    tail -n+2 |\
+    cut -d',' -f1 |\
+    sort |\
+    uniq
+))
+
+echo "[+] Benchmarking ${#PALLETS[@]} pallets for runtime $runtime"
+
+# Define the error file.
+ERR_FILE="${ARTIFACTS_DIR}/benchmarking_errors.txt"
+# Delete the error file before each run.
+rm -f $ERR_FILE
+
+# Benchmark each pallet.
+for PALLET in "${PALLETS[@]}"; do
+  echo "[+] Benchmarking $PALLET for $runtime";
+
+  output_file=""
+  if [[ $PALLET == *"::"* ]]; then
+    # translates e.g. "pallet_foo::bar" to "pallet_foo_bar"
+    output_file="${PALLET//::/_}.rs"
+  fi
+
+  OUTPUT=$(
+    $POLKADOT_BIN benchmark pallet \
+    --chain="${runtime}-dev" \
+    --steps=50 \
+    --repeat=20 \
+    --no-storage-info \
+    --no-median-slopes \
+    --no-min-squares \
+    --pallet="$PALLET" \
+    --extrinsic="*" \
+    --execution=wasm \
+    --wasm-execution=compiled \
+    --header="$output_path/file_header.txt" \
+    --output="$output_path/runtime/${runtime}/src/weights/${output_file}" 2>&1
+  )
+  if [ $? -ne 0 ]; then
+    echo "$OUTPUT" >> "$ERR_FILE"
+    echo "[-] Failed to benchmark $PALLET. Error written to $ERR_FILE; continuing..."
+  fi
+done
+
+# Check if the error file exists.
+if [ -f "$ERR_FILE" ]; then
+  echo "[-] Some benchmarks failed. See: $ERR_FILE"
+else
+  echo "[+] All benchmarks passed."
+fi

scripts/lib/bench-all-substrate.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+# This file is part of Substrate.
+# Copyright (C) 2022 Parity Technologies (UK) Ltd.
+# SPDX-License-Identifier: Apache-2.0
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script has three parts which all use the Substrate runtime:
+# - Pallet benchmarking to update the pallet weights
+# - Overhead benchmarking for the Extrinsic and Block weights
+# - Machine benchmarking
+#
+# Should be run on a reference machine to gain accurate benchmarks
+# current reference machine: https://github.com/paritytech/substrate/pull/5848
+
+# Original source: https://github.com/paritytech/substrate/blob/ff9921a260a67e3a71f25c8b402cd5c7da787a96/scripts/run_all_benchmarks.sh
+# Fail if any sub-command in a pipe fails, not just the last one.
+set -o pipefail
+# Fail on undeclared variables.
+set -u
+# Fail if any sub-command fails.
+set -e
+# Fail on traps.
+# set -E
+
+# default RUST_LOG is warn, but could be overridden
+export RUST_LOG="${RUST_LOG:-error}"
+
+echo "[+] Compiling Substrate benchmarks..."
+cargo build --profile=$profile --locked --features=runtime-benchmarks -p staging-node-cli
+
+# The executable to use.
+SUBSTRATE="./target/$profile/substrate-node"
+
+# Manually exclude some pallets.
+EXCLUDED_PALLETS=(
+  # Helper pallets
+  "pallet_election_provider_support_benchmarking"
+  # Pallets without automatic benchmarking
+  "pallet_babe"
+  "pallet_grandpa"
+  "pallet_mmr"
+  "pallet_offences"
+  # Only used for testing, does not need real weights.
+  "frame_benchmarking_pallet_pov"
+  "pallet_example_tasks"
+  "pallet_example_basic"
+  "pallet_example_split"
+  "pallet_example_kitchensink"
+  "pallet_example_mbm"
+  "tasks_example"
+)
+
+# Load all pallet names in an array.
+ALL_PALLETS=($(
+  $SUBSTRATE benchmark pallet --list --chain=dev |\
+    tail -n+2 |\
+    cut -d',' -f1 |\
+    sort |\
+    uniq
+))
+
+# Define the error file.
+ERR_FILE="${ARTIFACTS_DIR}/benchmarking_errors.txt"
+
+# Delete the error file before each run.
+rm -f "$ERR_FILE"
+
+mkdir -p "$(dirname "$ERR_FILE")"
+
+# Update the block and extrinsic overhead weights.
+echo "[+] Benchmarking block and extrinsic overheads..."
+OUTPUT=$(
+  $SUBSTRATE benchmark overhead \
+  --chain=dev \
+  --wasm-execution=compiled \
+  --weight-path="$output_path/frame/support/src/weights/" \
+  --header="$output_path/HEADER-APACHE2" \
+  --warmup=10 \
+  --repeat=100 2>&1
+)
+if [ $? -ne 0 ]; then
+  echo "$OUTPUT" >> "$ERR_FILE"
+  echo "[-] Failed to benchmark the block and extrinsic overheads. Error written to $ERR_FILE; continuing..."
+fi
+
+echo "[+] Benchmarking ${#ALL_PALLETS[@]} Substrate pallets and excluding ${#EXCLUDED_PALLETS[@]}."
+
+echo "[+] Excluded pallets ${EXCLUDED_PALLETS[@]}"
+echo "[+] ------ "
+echo "[+] Whole list pallets ${ALL_PALLETS[@]}"
+
+# Benchmark each pallet.
+for PALLET in "${ALL_PALLETS[@]}"; do
+  FOLDER="$(echo "${PALLET#*_}" | tr '_' '-')";
+  WEIGHT_FILE="$output_path/frame/${FOLDER}/src/weights.rs"
+
+   # Skip the pallet if it is in the excluded list.
+
+  if [[ " ${EXCLUDED_PALLETS[@]} " =~ " ${PALLET} " ]]; then
+    echo "[+] Skipping $PALLET as it is in the excluded list."
+    continue
+  fi
+
+  echo "[+] Benchmarking $PALLET with weight file $WEIGHT_FILE";
+
+  set +e # Disable exit on error for the benchmarking of the pallets
+  OUTPUT=$(
+    $SUBSTRATE benchmark pallet \
+    --chain=dev \
+    --steps=50 \
+    --repeat=20 \
+    --pallet="$PALLET" \
+    --no-storage-info \
+    --no-median-slopes \
+    --no-min-squares \
+    --extrinsic="*" \
+    --wasm-execution=compiled \
+    --heap-pages=4096 \
+    --output="$WEIGHT_FILE" \
+    --header="$output_path/HEADER-APACHE2" \
+    --template="$output_path/.maintain/frame-weight-template.hbs" 2>&1
+  )
+  if [ $? -ne 0 ]; then
+    echo -e "$PALLET: $OUTPUT\n" >> "$ERR_FILE"
+    echo "[-] Failed to benchmark $PALLET. Error written to $ERR_FILE; continuing..."
+  fi
+  set -e # Re-enable exit on error
+done
+
+
+# Check if the error file exists.
+if [ -s "$ERR_FILE" ]; then
+  echo "[-] Some benchmarks failed. See: $ERR_FILE"
+  exit 1
+else
+  echo "[+] All benchmarks passed."
+fi