Migrate some pallets to benchmark v2 (#6311) #6105
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tests misc | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| merge_group: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| # Jobs in this workflow depend on each other, only for limiting peak amount of spawned workers | |
| jobs: | |
| preflight: | |
| uses: ./.github/workflows/reusable-preflight.yml | |
| # more information about this job can be found here: | |
| # https://github.com/paritytech/substrate/pull/3778 | |
| test-full-crypto-feature: | |
| needs: [preflight] | |
| runs-on: ${{ needs.preflight.outputs.RUNNER }} | |
| if: ${{ needs.preflight.outputs.changes_rust }} | |
| timeout-minutes: 60 | |
| container: | |
| image: ${{ needs.preflight.outputs.IMAGE }} | |
| env: | |
| # Enable debug assertions since we are running optimized builds for testing | |
| # but still want to have debug assertions. | |
| RUSTFLAGS: "-C debug-assertions" | |
| RUST_BACKTRACE: 1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| cd substrate/primitives/core/ | |
| forklift cargo build --locked --no-default-features --features full_crypto | |
| cd ../application-crypto | |
| forklift cargo build --locked --no-default-features --features full_crypto | |
| test-frame-examples-compile-to-wasm: | |
| timeout-minutes: 20 | |
| # into one job | |
| needs: [preflight, test-full-crypto-feature] | |
| runs-on: ${{ needs.preflight.outputs.RUNNER }} | |
| if: ${{ needs.preflight.outputs.changes_rust }} | |
| container: | |
| image: ${{ needs.preflight.outputs.IMAGE }} | |
| env: | |
| # Enable debug assertions since we are running optimized builds for testing | |
| # but still want to have debug assertions. | |
| RUSTFLAGS: "-C debug-assertions" | |
| RUST_BACKTRACE: 1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| cd substrate/frame/examples/offchain-worker/ | |
| forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features | |
| cd ../basic | |
| forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features | |
| test-frame-ui: | |
| timeout-minutes: 60 | |
| needs: [preflight] | |
| runs-on: ${{ needs.preflight.outputs.RUNNER }} | |
| if: ${{ needs.preflight.outputs.changes_rust }} | |
| container: | |
| image: ${{ needs.preflight.outputs.IMAGE }} | |
| env: | |
| # Enable debug assertions since we are running optimized builds for testing | |
| # but still want to have debug assertions. | |
| RUSTFLAGS: "-C debug-assertions -D warnings" | |
| RUST_BACKTRACE: 1 | |
| SKIP_WASM_BUILD: 1 | |
| # Ensure we run the UI tests. | |
| RUN_UI_TESTS: 1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| cargo version | |
| forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental ui | |
| forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental ui | |
| forklift cargo test --locked -q --profile testnet -p xcm-procedural ui | |
| forklift cargo test --locked -q --profile testnet -p frame-election-provider-solution-type ui | |
| forklift cargo test --locked -q --profile testnet -p sp-api-test ui | |
| # There is multiple version of sp-runtime-interface in the repo. So we point to the manifest. | |
| forklift cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml ui | |
| test-deterministic-wasm: | |
| timeout-minutes: 20 | |
| needs: [preflight, test-frame-examples-compile-to-wasm] | |
| runs-on: ${{ needs.preflight.outputs.RUNNER }} | |
| if: ${{ needs.preflight.outputs.changes_rust }} | |
| container: | |
| image: ${{ needs.preflight.outputs.IMAGE }} | |
| env: | |
| WASM_BUILD_NO_COLOR: 1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| # build runtime | |
| forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime | |
| # make checksum | |
| sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256 | |
| cargo clean | |
| # build again | |
| forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime | |
| # confirm checksum | |
| sha256sum -c checksum.sha256 | |
| cargo-check-benches: | |
| needs: [preflight] | |
| if: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }} | |
| timeout-minutes: 60 | |
| strategy: | |
| matrix: | |
| branch: [master, current] | |
| runs-on: ${{ needs.preflight.outputs.RUNNER }} | |
| container: | |
| image: ${{ needs.preflight.outputs.IMAGE }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| # if branch is master, use the branch, otherwise set empty string, so it uses the current context | |
| # either PR (including forks) or merge group (main repo) | |
| ref: ${{ matrix.branch == 'master' && matrix.branch || '' }} | |
| - name: script | |
| run: | | |
| ARTIFACTS_DIR=./artifacts | |
| BENCH_TRIE_READ=::trie::read::small | |
| BENCH_NODE_IMPORT=::node::import::sr25519::transfer_keep_alive::paritydb::small | |
| mkdir -p $ARTIFACTS_DIR | |
| SKIP_WASM_BUILD=1 forklift cargo check --locked --benches --all; | |
| forklift cargo run --locked --release -p node-bench -- $BENCH_TRIE_READ --json | tee $ARTIFACTS_DIR/bench_trie_read_small.json; | |
| forklift cargo run --locked --release -p node-bench -- $BENCH_NODE_IMPORT --json | tee $ARTIFACTS_DIR/bench_transfer_keep_alive.json | |
| - name: Upload artifacts | |
| uses: actions/[email protected] | |
| with: | |
| path: ./artifacts | |
| name: cargo-check-benches-${{ matrix.branch }}-${{ github.sha }} | |
| retention-days: 1 | |
| node-bench-regression-guard: | |
| timeout-minutes: 20 | |
| if: always() && !cancelled() | |
| runs-on: ubuntu-latest | |
| needs: [preflight, cargo-check-benches] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download artifact (master run) | |
| uses: actions/[email protected] | |
| with: | |
| name: cargo-check-benches-master-${{ github.sha }} | |
| path: ./artifacts/master | |
| - name: Download artifact (current run) | |
| uses: actions/[email protected] | |
| with: | |
| name: cargo-check-benches-current-${{ github.sha }} | |
| path: ./artifacts/current | |
| - name: script | |
| id: compare | |
| run: | | |
| if [ "${{ github.ref_name }}" = "master" ]; then | |
| echo -e "Exiting on master branch" | |
| exit 0 | |
| fi | |
| docker run --rm \ | |
| -v $PWD/artifacts/master:/artifacts/master \ | |
| -v $PWD/artifacts/current:/artifacts/current \ | |
| paritytech/node-bench-regression-guard:latest \ | |
| node-bench-regression-guard --reference /artifacts/master --compare-with /artifacts/current | |
| if [ $? -ne 0 ]; then | |
| FAILED_MSG='### node-bench-regression-guard failed ❌, check the regression in *cargo-check-benches* job' | |
| echo $FAILED_MSG | |
| echo $FAILED_MSG >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| else | |
| echo "### node-bench-regression-guard passed ✅" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| test-node-metrics: | |
| needs: [preflight] | |
| timeout-minutes: 30 | |
| runs-on: ${{ needs.preflight.outputs.RUNNER }} | |
| if: ${{ needs.preflight.outputs.changes_rust }} | |
| container: | |
| image: ${{ needs.preflight.outputs.IMAGE }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run tests | |
| id: tests | |
| env: | |
| RUST_TOOLCHAIN: stable | |
| # Enable debug assertions since we are running optimized builds for testing | |
| # but still want to have debug assertions. | |
| RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" | |
| run: | | |
| forklift cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker --profile testnet --verbose --locked | |
| mkdir -p ./artifacts | |
| forklift cargo test --profile testnet --locked --features=runtime-metrics -p polkadot-node-metrics > ./artifacts/log.txt | |
| echo "Metrics test passed" | |
| - name: Upload artifacts if failed | |
| if: ${{ steps.tests.outcome != 'success' }} | |
| uses: actions/[email protected] | |
| with: | |
| name: node-metrics-failed | |
| path: ./artifacts | |
| # more information about this job can be found here: | |
| # https://github.com/paritytech/substrate/pull/6916 | |
| check-tracing: | |
| timeout-minutes: 20 | |
| needs: [preflight, test-node-metrics] | |
| runs-on: ${{ needs.preflight.outputs.RUNNER }} | |
| if: ${{ needs.preflight.outputs.changes_rust }} | |
| container: | |
| image: ${{ needs.preflight.outputs.IMAGE }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| forklift cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features | |
| forklift cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features --features=with-tracing | |
| check-metadata-hash: | |
| timeout-minutes: 20 | |
| needs: [preflight, check-tracing] | |
| runs-on: ${{ needs.preflight.outputs.RUNNER }} | |
| if: ${{ needs.preflight.outputs.changes_rust }} | |
| container: | |
| image: ${{ needs.preflight.outputs.IMAGE }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: script | |
| run: | | |
| forklift cargo build --locked -p westend-runtime --features metadata-hash | |
| # disabled until https://github.com/paritytech/polkadot-sdk/issues/5812 is resolved | |
| # cargo-hfuzz: | |
| # timeout-minutes: 20 | |
| # needs: [preflight, check-metadata-hash] | |
| # runs-on: ${{ needs.preflight.outputs.RUNNER }} | |
| # container: | |
| # image: ${{ needs.preflight.outputs.IMAGE }} | |
| # env: | |
| # # max 10s per iteration, 60s per file | |
| # HFUZZ_RUN_ARGS: | | |
| # --exit_upon_crash | |
| # --exit_code_upon_crash 1 | |
| # --timeout 10 | |
| # --run_time 60 | |
| # # use git version of honggfuzz-rs until v0.5.56 is out, we need a few recent changes: | |
| # # https://github.com/rust-fuzz/honggfuzz-rs/pull/75 to avoid breakage on debian | |
| # # https://github.com/rust-fuzz/honggfuzz-rs/pull/81 fix to the above pr | |
| # # https://github.com/rust-fuzz/honggfuzz-rs/pull/82 fix for handling absolute CARGO_TARGET_DIR | |
| # HFUZZ_BUILD_ARGS: | | |
| # --config=patch.crates-io.honggfuzz.git="https://github.com/altaua/honggfuzz-rs" | |
| # --config=patch.crates-io.honggfuzz.rev="205f7c8c059a0d98fe1cb912cdac84f324cb6981" | |
| # steps: | |
| # - name: Checkout | |
| # uses: actions/checkout@v4 | |
| # - name: Run honggfuzz | |
| # run: | | |
| # cd substrate/primitives/arithmetic/fuzzer | |
| # forklift cargo hfuzz build | |
| # for target in $(cargo read-manifest | jq -r '.targets | .[] | .name'); | |
| # do | |
| # forklift cargo hfuzz run "$target" || { printf "fuzzing failure for %s\n" "$target"; exit 1; }; | |
| # done | |
| # - name: Upload artifacts | |
| # uses: actions/[email protected] | |
| # with: | |
| # path: substrate/primitives/arithmetic/fuzzer/hfuzz_workspace/ | |
| # name: hfuzz-${{ github.sha }} | |
| cargo-check-each-crate: | |
| timeout-minutes: 70 | |
| needs: [preflight] | |
| runs-on: ${{ needs.preflight.outputs.RUNNER }} | |
| if: ${{ needs.preflight.outputs.changes_rust }} | |
| container: | |
| image: ${{ needs.preflight.outputs.IMAGE }} | |
| env: | |
| RUSTFLAGS: "-D warnings" | |
| CI_JOB_NAME: cargo-check-each-crate | |
| strategy: | |
| matrix: | |
| index: [1, 2, 3, 4, 5, 6, 7] # 7 parallel jobs | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Check Rust | |
| run: | | |
| rustup show | |
| rustup +nightly show | |
| - name: script | |
| run: | | |
| mkdir -p /github/home/.forklift | |
| cp .forklift/config.toml /github/home/.forklift/config.toml | |
| PYTHONUNBUFFERED=x .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }} | |
| cargo-check-all-crate-macos: | |
| timeout-minutes: 30 | |
| needs: [preflight] | |
| runs-on: ${{ needs.preflight.outputs.RUNNER_MACOS }} | |
| if: ${{ needs.preflight.outputs.changes_rust }} | |
| env: | |
| SKIP_WASM_BUILD: 1 | |
| steps: | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| - name: Set rust version from env file | |
| run: | | |
| RUST_VERSION=$(cat .github/env | sed -E 's/.*ci-unified:([^-]+)-([^-]+).*/\2/') | |
| echo $RUST_VERSION | |
| echo "RUST_VERSION=${RUST_VERSION}" >> $GITHUB_ENV | |
| - name: Set up Homebrew | |
| uses: Homebrew/actions/setup-homebrew@1ccc07ccd54b6048295516a3eb89b192c35057dc # master from 12.09.2024 | |
| - name: Install rust ${{ env.RUST_VERSION }} | |
| uses: actions-rust-lang/setup-rust-toolchain@11df97af8e8102fd60b60a77dfbf58d40cd843b8 # v1.10.1 | |
| with: | |
| cache: false | |
| toolchain: ${{ env.RUST_VERSION }} | |
| target: wasm32-unknown-unknown | |
| components: cargo, clippy, rust-docs, rust-src, rustfmt, rustc, rust-std | |
| - name: Install protobuf | |
| run: brew install protobuf | |
| - name: cargo info | |
| run: | | |
| echo "######## rustup show ########" | |
| rustup show | |
| echo "######## cargo --version ########" | |
| cargo --version | |
| - name: Run cargo check | |
| run: cargo check --workspace --locked | |
| confirm-required-test-misc-jobs-passed: | |
| runs-on: ubuntu-latest | |
| name: All test misc tests passed | |
| # If any new job gets added, be sure to add it to this array | |
| needs: | |
| - test-full-crypto-feature | |
| - test-frame-examples-compile-to-wasm | |
| - test-frame-ui | |
| - cargo-check-benches | |
| - node-bench-regression-guard | |
| - test-node-metrics | |
| - check-tracing | |
| - cargo-check-each-crate | |
| - test-deterministic-wasm | |
| - cargo-check-all-crate-macos | |
| # - cargo-hfuzz remove from required for now, as it's flaky | |
| if: always() && !cancelled() | |
| steps: | |
| - run: | | |
| tee resultfile <<< '${{ toJSON(needs) }}' | |
| FAILURES=$(cat resultfile | grep '"result": "failure"' | wc -l) | |
| if [ $FAILURES -gt 0 ]; then | |
| echo "### At least one required job failed ❌" >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| else | |
| echo '### Good job! All the required jobs passed 🚀' >> $GITHUB_STEP_SUMMARY | |
| fi |