diff --git a/bench/process/collect.jq b/bench/process/collect.jq
deleted file mode 100644
index 467b08eb7c0..00000000000
--- a/bench/process/collect.jq
+++ /dev/null
@@ -1,12 +0,0 @@
-def standard_run_desc(filename; sha256; format; ctime; mtime; branch; commit; config_name; iter; data):
- { filename: filename
- , sha256: sha256
- , format: format
- , ctime: ctime
- , mtime: mtime
- , branch: branch
- , commit: commit
- , config_name: config_name
- , iter: iter
- , data: data
- };
diff --git a/bench/process/membenches_v1.jq b/bench/process/membenches_v1.jq
deleted file mode 100644
index 6d67ad9f514..00000000000
--- a/bench/process/membenches_v1.jq
+++ /dev/null
@@ -1,47 +0,0 @@
-def format_specs:
-
-{ config_specs:
- [{ key: "config", header: "Config name"
- , path: ["config_name"]}
- ,{ key: "flags", header: "RTS Flags"
- , path: ["data", "flags"]
- }
- ,{ key: "failed", header: "Failed"
- , path: ["data", "failed"]
- # , post: not
- }]
-, result_specs:
- [{ key: "Average-RSS", header: "Average RSS", unit: "MB"
- , path: ["RSS", "avg"]
- , round: true
- }
- ,{ key: "Peak-RSS", header: "Peak RSS", unit: "MB"
- , path: ["RSS", "max"]
- , round: true
- }
- ,{ key: "Average-RTS-heap", header: "Average RTS heap", unit: "MB"
- , path: ["Heap", "avg"]
- , round: true
- }
- ,{ key: "Peak-RTS-heap", header: "Peak RTS heap", unit: "MB"
- , path: ["Heap", "max"]
- , round: true
- }
- ,{ key: "Wall-clock-time", header: "Wall clock time", unit: "s"
- , path: ["totaltime"]
- , round: true
- }
- ,{ key: "OS-CPU-time", header: "OS CPU time", unit: "s"
- , path: ["CentiCpuMax"]
- , round: true, scale: 100
- }
- ,{ key: "RTS-mutator-time", header: "RTS mutator time", unit: "s"
- , path: ["CentiMutMax"]
- , round: true, scale: 100
- }
- ,{ key: "RTS-GC-time", header: "RTS GC time", unit: "s"
- , path: ["SecGC"]
- , round: true
- }
- ]
-};
diff --git a/bench/process/process.jq b/bench/process/process.jq
deleted file mode 100644
index c19037b5583..00000000000
--- a/bench/process/process.jq
+++ /dev/null
@@ -1,54 +0,0 @@
-##
-## Aggregation
-##
-def mean: if length == 0
- then 0
- else reduce .[] as $n (0; . + $n) / length end;
-def pow2: . * .;
-def variance: . | mean as $mean | map_values(. - $mean | pow2) | mean;
-def stddev: . | (variance ?) | sqrt;
-
-def samples_to_variable (n):
- stddev as $stddev
- | (add / n) as $mean
- | { "mean": $mean
- , "stddev": $stddev
- , "relstddev": (if $stddev == null then 0 else $stddev / $mean end)
- , "raw": .
- };
-
-def varspec_to_variable (objects; nobjects):
- .key as $key
- | .path as $path
- | objects
- | { "\($key)":
- map (.data
- | getpath($path)
- as $val
- | if $val != null then $val
- else error("Path \($path) unreachable among top level keys: \(keys)")
- end
- )
- | samples_to_variable(nobjects)
- };
-
-def description_from_headliner(x; rest):
- map ( . as $spec
- | (x | getpath($spec.path)) as $head
- | (rest | map (getpath($spec.path) == $head) | all) as $coherence
- | { "\($spec.key)":
- (if $coherence == true
- then $head
- else error("Incoherence on config key: \($spec.key)")
- end) })
- | add;
-
-def aggregate_config_runs_variables (config_specs; result_specs):
- . as $runs
- | .[0] as $headliner
- | length as $nruns
- | result_specs
- | map(varspec_to_variable($runs; $nruns))
- | add
- | (config_specs | description_from_headliner($headliner; $runs[1:]))
- + .;
diff --git a/bench/process/process.sh b/bench/process/process.sh
deleted file mode 100755
index a939158ee10..00000000000
--- a/bench/process/process.sh
+++ /dev/null
@@ -1,144 +0,0 @@
-#!/bin/sh
-# shellcheck disable=SC1090
-set -eu
-
-fail() {
- echo -e "ERROR: $*" >&2
- exit 1
-}
-
-default_op='report'
-default_format='csv'
-
-format=$default_format
-header_footer='false'
-no_progress=
-
-while test $# -ge 1
-do case "$1" in
- --github ) format='github';;
- --csv ) format='csv'; header_footer='false';;
- --cls ) echo -en "\ec" >&2;;
- --no-progress ) no_progress='true';;
- --trace ) set -x;;
- * ) break;; esac; shift; done
-
-op=${1:-$default_op}; shift || true
-
-main() {
- PROCESS_ORIG_PWD=$(pwd)
- pushd "$(dirname $0)" >/dev/null || exit 1
- case "${op}" in
- collect ) op_collect "$@";;
- process ) op_process;;
- render ) op_render;;
- render-html )
- op_render_html;;
- render-hydra-charts )
- op_render_hydra_charts "$@";;
-
- report ) op_collect "$@" | op_process | op_render;;
-
- call ) eval "$@";;
- * ) echo "ERROR: operation must be one of: collect process render render-hydra-charts report" >&2; exit 1;; esac
-}
-
-hardcoded_branch='membench'
-hardcoded_commit='a7ee17d1af44b571c6e476916bd24ed65db97e15'
-
-function op_collect() {
- local desc=${1?-USAGE: $0 collect DESCRIPTION [FORMAT] [DIR]}
- local format=${2:-membenches_v1}
- local literal_dir=${3:-.}
-
- local dir=$(pushd "$PROCESS_ORIG_PWD" >/dev/null; realpath "$literal_dir")
- test -d "${dir}" -a -n "$(ls "${dir}"/*/*.json)" ||
- fail "${literal_dir} (realpath $dir) must be a writable directory with subdirectories containing JSON files with ${format} output schema"
-
- test -n "$no_progress" || echo -ne "Collecting runs in $dir: " >&2
- local args_global=(
- --arg desc "$desc"
- --arg format "$format"
- --arg now "$(date --utc --iso-8601=seconds)"
- )
-case $format in
- membenches_v1 )
- {
- for f in $dir/*/refined.json
- do local fpad="$f "
- test -n "$no_progress" || echo -n "$fpad" >&2
- local args_run=(
- --arg filename "$f"
- --arg format "$format"
- --arg sha256 "$(sha256sum $f | cut -d' ' -f1)"
- --arg ctime "$(stat --format=%w $f)"
- --arg mtime "$(stat --format=%y $f)"
- --arg config "$(echo $f | xargs dirname | xargs basename | cut -d- -f1)"
- --arg iter "$(echo $f | xargs dirname | xargs basename | cut -d- -f2)"
- --arg hardcoded_branch $hardcoded_branch
- --arg hardcoded_commit $hardcoded_commit
- --slurpfile data "$f"
- )
- jq 'include "collect";
-
- standard_run_desc($filename; $sha256; $format; $ctime; $mtime; $hardcoded_branch; $hardcoded_commit; $config; $iter; $data[0])
- ' "$f" "${args_global[@]}" "${args_run[@]}"
- test -n "$no_progress" || printf "${fpad//?/\\b}" >&2
- done
- test -n "$no_progress" || echo >&2; };;
- * )
- fail "unknown result format: $format"
-esac |
- jq 'include "'"$format"'";
-
- { description: $desc
- , format: $format
- , ctime: $now
- , runs: .
- }
- + format_specs
- ' --slurp "${args_global[@]}"
-}
-
-function op_process() {
- jq 'include "process";
-
- . as $batch
- | $batch.runs
- | group_by(.config_name)
- | map (aggregate_config_runs_variables ($batch.config_specs; $batch.result_specs))
- | $batch
- + { configs: . }
- '
-}
-
-function op_render() {
- jq 'include "render";
-
- render('"$header_footer"')
- ' --raw-output
-}
-
-function op_render_html() {
- jq 'include "render";
-
- render_html
- ' --raw-output
-}
-
-function op_render_hydra_charts() {
- local config='baseline'
- while test $# -ge 1
- do case "$1" in
- --config ) config=$2; shift;;
- * ) break;; esac; shift; done
- jq 'include "render";
-
- render_hydra_charts_for_config ("'$config'")
- ' --raw-output
-}
-
-###
-### Main
-###
-main "$@"
diff --git a/bench/process/render.jq b/bench/process/render.jq
deleted file mode 100644
index d51c767d3c3..00000000000
--- a/bench/process/render.jq
+++ /dev/null
@@ -1,214 +0,0 @@
-##
-## Presentation
-##
-
-def decimal_pt: (. * 10 | floor) / 10;
-def decimal_pt2: (. * 100 | floor) / 100;
-def float_n(width): "\(.)" | .[:width + (if .[0:1] == "-" then 1 else 0 end)];
-def downscale(factor): . / factor;
-
-## Expect name of a simple numeric field.
-def field(fname; f):
- .[fname] as $val
- | "\($val | f)";
-
-## Expect name of a rich variable.
-def var(fname; f):
- .[fname] as $val
- | "\($val.mean | f) | \("\($val.relstddev)" | .[:4])";
-
-def render_value(spec; val):
- (val.mean / (spec.scale // 1))
- | if spec.round
- then ceil
- else . end
- | tostring;
-
-def render_value_at_index(spec; val; ix):
- (val.raw[ix] / (spec.scale // 1))
- | if spec.round
- then ceil
- else . end
- | tostring;
-
-def render_value_relstddev(spec; val):
- (val.relstddev / (spec.scale // 1) * 100)
- | tostring
- | .[:5];
-
-def render_config_hydra_charts (res):
- . as $x
- | (res
- | map(. as $spec
- | $x[$spec.key] as $val
- # For each value, produce two Hydra metrics:
- # - (id, value, unit)
- # - (id-CoV, CoV, %)
- | [ $spec.key
- , render_value($spec; $val)
- , $spec.unit
- , $spec.key + "-coeff-variation"
- , render_value_relstddev($spec; $val)
- , "%"
- ]
- | join(" ")))
- as $properties
-
- | ($properties | join("\n"));
-
-def render_html_config_runs(cf; res; count):
- [ "
\n"
- , "Per-run raw data for config '\(.config)'"
- , ""
- , ((cf + res)
- | map(if .unit != null
- then [ .header + ", " + .unit ]
- else [ .header ] end)
- | add
- | join(" | "))
- , " |
\n"
- , . as $x
- |
- ( [range(count)]
- | map( . as $ix
- | cf + res
- | map(. as $spec
- | $x[$spec.key] as $val
- | if .unit != null
- then render_value_at_index($spec; $val; $ix)
- else $val | tostring end
- | "" + . + " | ")
- | add
- | ("" + . + "
\n"))
- | add)
- , "\n
"
- ]
- | add;
-
-def render_html_summary(cf; res):
- [ "\n"
- , "Summary stats for run"
- , ""
- , ((cf + res)
- | map(if .unit != null
- then [ .header + ", " + .unit
- , .header + ", " + "σ/μ"
- ]
- else [ .header ] end)
- | add
- | join(" | "))
- , " |
\n"
- , (.configs
- | map( . as $x
- | cf + res
- | map(. as $spec
- | $x[$spec.key] as $val
- | if .unit != null
- then [ render_value($spec; $val)
- , ""
- , render_value_relstddev($spec; $val) ]
- else [ $val | tostring ] end
- | [" | "] + . + [" | "]
- | add)
- | add
- | ("" + . + "
\n"))
- | add)
- , "\n
"
- ]
- | add;
-
-def render_html:
- .config_specs as $cf
- | .result_specs as $res
- | .runs as $runs
-
- | [render_html_summary($cf; $res)]
- + (.configs
- | map( . as $config
- | render_html_config_runs($cf; $res;
- $runs
- | map(select(.config_name == $config.config))
- | length)))
- | join("\n
\n");
-
-def render_config (format; cf; res):
- . as $x
- | (if format != "csv" then [null] else [] end
- +
- (cf
- | map($x[.key]))
- +
- (res
- | map(. as $spec
- | $x[$spec.key] as $val
- | [ render_value($spec; $val)
- , render_value_relstddev($spec; $val)])
- | add))
- as $columns
-
- | ($columns | join(" | "));
-
-def render_table_head (format; cf; res):
- .
- | (if format != "csv" then [null] else [] end
- +
- (cf
- | map(.header))
- +
- (res
- | map([.header, "σ/μ"])
- | add))
- as $columns
-
- | if format == "github"
- then [([null] + ($columns | map("--")) + [null])
- | join("|")] else [] end
- +
- [ $columns | join(" | ")];
-
-def render_table:
- .format as $format
- | .config_specs as $config_specs
- | .result_specs as $result_specs
- | render_table_head (.format; .config_specs; .result_specs)
- + (.configs | map (render_config ($format; $config_specs; $result_specs)));
-
-def add_header_footer(commits; run_counts; slot_counts):
- .
- | ([ "Parameters:\n"
- , if run_counts | all(. == run_counts[0])
- then " Every value is mean of \(run_counts[0]) runs,\n"
- else " Every value is mean of varying amount of runs (\(run_counts)).\n" end
- , if slot_counts | all(. == slot_counts[0])
- then " Each run was syncing \(slot_counts[0]) slots, or \(slot_counts[0] / 21600 | floor) epochs over loopback, from a quiescent server.\n"
- else " Runs were for varying amount of slots (\(slot_counts)).\n" end
- , "\nLegend:\n"
- , " wall = total_wall_seconds, total cpu = total_cpu_seconds\n"
- , " total alloc = allocated_bytes, copied = copied_bytes, max live = max_live_bytes\n"
- , " See https://github.com/ghc/ghc/blob/master/includes/RtsAPI.h for details.\n"
- , " Each value is followed by σ/μ, i.e. relative stddev (or CoV).\n"
- , " Δ% is change from baseline.\n"
- , "\n\n"]) + .;
- #| . + ["\n\n\(commits | map(" - \(.branch) / \(.commit) = https://github.com/'${github_user}'/cardano-node/tree/\(.commit)\n") | add)"];
-
-def render(header_footer):
- .commits as $commits
-| .run_counts as $run_counts
-| .slot_counts as $slot_counts
-| render_table
-| if header_footer == true
- then add_header_footer($commits; $run_counts; $slot_counts)
- else . end
-| join("\n");
-
-def render_hydra_charts_for_config(config_name):
- .result_specs as $result_specs
-| .configs
-| map(select(.config == config_name))
-| if . == null or . == []
- then error ("render_hydra_charts_for_config: unknown config \(config_name)")
- else .[0]
- end
-| render_config_hydra_charts ($result_specs)
-# | join("\n")
-;
diff --git a/flake.nix b/flake.nix
index 045555f7bc5..e8b28188559 100644
--- a/flake.nix
+++ b/flake.nix
@@ -203,11 +203,11 @@
## This is a very light profile, no caching&pinning needed.
workbench-ci-test =
- workbenchTest { profileName = "ci-test-hydra-bage";
+ workbenchTest { profileName = "ci-test-hydra-coay";
workbenchStartArgs = [ "--create-testnet-data" ];
};
workbench-ci-test-trace =
- workbenchTest { profileName = "ci-test-hydra-bage";
+ workbenchTest { profileName = "ci-test-hydra-coay";
workbenchStartArgs = [ "--create-testnet-data" "--trace" ];
};
diff --git a/nix/custom-config.nix b/nix/custom-config.nix
index 2390f9e10db..86618c6411d 100644
--- a/nix/custom-config.nix
+++ b/nix/custom-config.nix
@@ -13,17 +13,6 @@ self: {
extraBackendConfig = {};
useCabalRun = true;
};
- membench = {
- snapshotSlot = 37173650;
- finalChunkNo = 1800;
- rtsMemSize = null;
- rtsflags = "";
- limit2 = "6553M";
- variantTable = {
- baseline = { legacyTracing = false; };
- };
- shelleyGenesisHash = "1a3be38bcbb7911969283716ad7aa550250226b76a61fc51cc9a9a35d9276d81";
- };
# optional extra haskell.nix module
haskellNix = {};
}
diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix
index b29a57f74ca..f79b13367b1 100644
--- a/nix/nixos/cardano-node-service.nix
+++ b/nix/nixos/cardano-node-service.nix
@@ -128,10 +128,9 @@ let
];
};
instanceDbPath = cfg.databasePath i;
- utxoLmdbParams = ["--v1-lmdb-ledger-db-backend"]
+ utxoLmdbParams = ["--utxos-on-disk"]
++ lib.optionals (cfg.lmdbDatabasePath i != null)
- [ "--ssd-database-dir ${cfg.lmdbDatabasePath i}"
- # "--ssd-snapshot-tables"
+ [ "--utxos-database-path ${cfg.lmdbDatabasePath i}"
];
cmd = builtins.filter (x: x != "") [
"${cfg.executable} run"
diff --git a/nix/workbench/backend/runner.nix b/nix/workbench/backend/runner.nix
index c4bcde5f773..c183e4e02f5 100644
--- a/nix/workbench/backend/runner.nix
+++ b/nix/workbench/backend/runner.nix
@@ -117,6 +117,7 @@ in
moreutils
nix
pstree
+ pkgs.util-linux
workbench
zstd
]
diff --git a/nix/workbench/lib.sh b/nix/workbench/lib.sh
index 622db1bfcba..ece49fae54d 100644
--- a/nix/workbench/lib.sh
+++ b/nix/workbench/lib.sh
@@ -1,3 +1,5 @@
+# shellcheck shell=bash
+
to_jsonlist() {
for x in "$@"
do echo "\"$x\""
@@ -322,3 +324,19 @@ wait_internal () {
return 0
fi
}
+
+# Function to acquire a lock on a file. This is meant to be called in a
+# subshell, so that the lock is released when the subshell exits. E.g.
+#
+# (
+# acquire_lock
+# ...
+# )
+#
+acquire_lock() {
+ WB_LOCKFILE=${WB_LOCKFILE:-/tmp/workbench.lock}
+ info lockfile "$(white "waiting to acquire the lock on ${WB_LOCKFILE}")"
+ exec {lock_fd}>"$WB_LOCKFILE"
+ flock $lock_fd
+ info lockfile "$(green "lock acquired")"
+}
diff --git a/nix/workbench/run.sh b/nix/workbench/run.sh
index fd5e46b096c..d131f6b2f2e 100644
--- a/nix/workbench/run.sh
+++ b/nix/workbench/run.sh
@@ -572,7 +572,12 @@ EOF
## 8. deploy genesis
progress "run | genesis" "deploying.."
- backend deploy-genesis "$dir"
+ (
+ # This step is resource intensive so we use a lockfile to avoid
+ # running it in parallel to a benchmark.
+ acquire_lock
+ backend deploy-genesis "$dir"
+ )
## 9. everything needed to start-[tracers|nodes|generator] should be
## ready
@@ -814,7 +819,12 @@ EOF
local scenario=${scenario_override:-$(jq -r .scenario "$dir"/profile.json)}
scenario "$scenario" "$dir"
- backend fetch-logs "$dir"
+ (
+ # This step is resource intensive so we use a lockfile to avoid
+ # running it in parallel to a benchmark.
+ acquire_lock
+ backend fetch-logs "$dir"
+ )
backend stop-cluster "$dir"
run compat-meta-fixups "$run"
diff --git a/nix/workbench/scenario.sh b/nix/workbench/scenario.sh
index d8572585b50..6ea13f2b992 100644
--- a/nix/workbench/scenario.sh
+++ b/nix/workbench/scenario.sh
@@ -150,7 +150,12 @@ scenario_exit_trap() {
echo >&2
msg "scenario: $(with_color yellow exit trap triggered)"
backend stop-all "$__scenario_exit_trap_dir"
- backend fetch-logs "$__scenario_exit_trap_dir"
+ (
+ # This step is resource intensive so we use a lockfile to avoid
+ # running it in parallel to a benchmark.
+ acquire_lock
+ backend fetch-logs "$__scenario_exit_trap_dir"
+ )
backend stop-cluster "$__scenario_exit_trap_dir"
msg "scenario: $(with_color yellow exit trap finished)"
}
diff --git a/nix/workbench/shell.nix b/nix/workbench/shell.nix
index 62079d46af8..5f5593d1b9d 100644
--- a/nix/workbench/shell.nix
+++ b/nix/workbench/shell.nix
@@ -129,6 +129,7 @@ in project.shellFor {
pkgs.moreutils
pkgs.pstree
pkgs.time
+ pkgs.util-linux
workbench-runner.workbench-interactive-start
workbench-runner.workbench-interactive-stop
workbench-runner.workbench-interactive-restart
diff --git a/nix/workbench/wb b/nix/workbench/wb
index 1aa94ac0290..dd0565df53d 100755
--- a/nix/workbench/wb
+++ b/nix/workbench/wb
@@ -219,7 +219,13 @@ start() {
for ((top_i=0; top_i