Add support for memory-profiling on subsystem-bench (paritytech#5522)

Add support in subsystem-benchmarks to profile memory usage using the
jemalloc builting profiler, this allows us to run each benchmark with
profiling enabled and determine if the memory usage patters are in
conformance with our expectations.

---------

Signed-off-by: Alexandru Gheorghe <[email protected]>

Cargo.toml

@@ -810,6 +810,7 @@ is-terminal = { version = "0.4.9" }
 is_executable = { version = "1.0.1" }
 isahc = { version = "1.2" }
 itertools = { version = "0.11" }
+jemalloc_pprof = { version = "0.4" }
 jobserver = { version = "0.1.26" }
 jsonpath_lib = { version = "0.3" }
 jsonrpsee = { version = "0.24.3" }

polkadot/node/subsystem-bench/Cargo.toml

@@ -19,7 +19,11 @@ path = "src/cli/subsystem-bench.rs"
 # Prevent rustdoc error. Already documented from top-level Cargo.toml.
 doc = false
 
+
 [dependencies]
+tikv-jemallocator = { features = ["profiling", "unprefixed_malloc_on_supported_platforms"], workspace = true, optional = true }
+jemalloc_pprof = { workspace = true, optional = true }
+polkadot-service = { workspace = true, default-features = true }
 polkadot-node-subsystem = { workspace = true, default-features = true }
 polkadot-node-subsystem-util = { workspace = true, default-features = true }
 polkadot-node-subsystem-types = { workspace = true, default-features = true }
@@ -93,3 +97,7 @@ strum = { features = ["derive"], workspace = true, default-features = true }
 
 [features]
 default = []
+memprofile = [
+	"dep:jemalloc_pprof",
+	"dep:tikv-jemallocator",
+]

polkadot/node/subsystem-bench/README.md

@@ -260,6 +260,41 @@ This file is best interpreted with `cg_annotate --auto=yes cachegrind.out.<pid>`
 
 For finer profiling of cache misses, better use `perf` on a bare-metal machine.
 
+### Profile memory usage using jemalloc
+
+Bellow you can find instructions how to setup and run profiling with jemalloc, this is complementary
+with using other memory profiling tools like: <https://github.com/koute/bytehound?tab=readme-ov-file#basic-usage>.
+
+#### Prerequisites
+
+Install tooling with:
+
+```
+sudo apt install libjemalloc-dev graphviz
+```
+
+#### Generate memory usage snapshots
+
+Memory usage can be profiled by running any subsystem benchmark with `--features memprofile`, e.g:
+
+```
+RUSTFLAGS=-g cargo run -p polkadot-subsystem-bench --release --features memprofile -- polkadot/node/subsystem-bench/examples/approvals_throughput.yaml
+```
+
+#### Interpret the results
+
+After the benchmark ran the memory usage snapshots can be found in `/tmp/subsystem-bench*`, to extract the information
+from a snapshot you can use `jeprof` like this:
+
+```
+jeprof --text PATH_TO_EXECUTABLE_WITH_DEBUG_SYMBOLS /tmp/subsystem-bench.1222895.199.i199.heap > statistics.txt
+```
+
+Useful links:
+
+- Tutorial: <https://www.magiroux.com/rust-jemalloc-profiling/>
+- Jemalloc configuration options: <https://jemalloc.net/jemalloc.3.html>
+
 ## Create new test objectives
 
 This tool is intended to make it easy to write new test objectives that focus individual subsystems,

polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs

@@ -182,6 +182,17 @@ impl BenchCli {
 	}
 }
 
+#[cfg(feature = "memprofile")]
+#[global_allocator]
+static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
+#[cfg(feature = "memprofile")]
+#[allow(non_upper_case_globals)]
+#[export_name = "malloc_conf"]
+// See https://jemalloc.net/jemalloc.3.html for more information on the configuration options.
+pub static malloc_conf: &[u8] =
+	b"prof:true,prof_active:true,lg_prof_interval:30,lg_prof_sample:21,prof_prefix:/tmp/subsystem-bench\0";
+
 fn main() -> eyre::Result<()> {
 	color_eyre::install()?;
 	sp_tracing::try_init_simple();