From 3209317bed4904ee167cc1e021acd9355c91c50f Mon Sep 17 00:00:00 2001 From: Povilas Liubauskas Date: Tue, 25 Jun 2024 11:34:23 +0300 Subject: [PATCH 1/2] Only prepare execution payload if there is a registered validator proposing in the next slot. Add AlwaysPrepareExecutionPayload feature to prepare execution in advance for each slot. --- features/src/lib.rs | 1 + grandine-snapshot-tests | 2 +- operation_pools/src/attestation_agg_pool/manager.rs | 12 +++++++++++- validator/src/validator.rs | 7 +++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/features/src/lib.rs b/features/src/lib.rs index 9438e7e2..cabf0214 100644 --- a/features/src/lib.rs +++ b/features/src/lib.rs @@ -39,6 +39,7 @@ static FEATURES: [AtomicBool; Feature::CARDINALITY] = [FALSE; Feature::CARDINALI )] pub enum Feature { AlwaysPrepackAttestations, + AlwaysPrepareExecutionPayload, CacheTargetStates, DebugEth1, DebugP2p, diff --git a/grandine-snapshot-tests b/grandine-snapshot-tests index 9a2f3ca7..ca7f8884 160000 --- a/grandine-snapshot-tests +++ b/grandine-snapshot-tests @@ -1 +1 @@ -Subproject commit 9a2f3ca7cee292d64533c4b6f448083066e6a0db +Subproject commit ca7f888404c36a5ad816bf93d76a0e5fb4e6249f diff --git a/operation_pools/src/attestation_agg_pool/manager.rs b/operation_pools/src/attestation_agg_pool/manager.rs index 4cc840c9..193e65fe 100644 --- a/operation_pools/src/attestation_agg_pool/manager.rs +++ b/operation_pools/src/attestation_agg_pool/manager.rs @@ -1,3 +1,4 @@ +use core::ops::RangeBounds; use std::sync::Arc; use anyhow::{Context, Error, Result}; @@ -15,7 +16,7 @@ use types::{ config::Config, phase0::{ containers::{Attestation, AttestationData}, - primitives::{Epoch, ValidatorIndex, H256}, + primitives::{Epoch, Slot, ValidatorIndex, H256}, }, preset::Preset, }; @@ -133,6 +134,15 @@ impl Manager { }); } + pub async fn has_registered_validators_proposing_in_slots( + &self, + range: impl RangeBounds + Send, + ) -> bool { + self.pool + .has_registered_validators_proposing_in_slots(range) + .await + } + pub fn pack_proposable_attestations(&self) { self.spawn_detached(PackProposableAttestationsTask { pool: self.pool.clone_arc(), diff --git a/validator/src/validator.rs b/validator/src/validator.rs index 9e07e649..ca203166 100644 --- a/validator/src/validator.rs +++ b/validator/src/validator.rs @@ -320,6 +320,13 @@ impl Validator { } }, ValidatorMessage::PrepareExecutionPayload(slot, safe_execution_payload_hash, finalized_execution_payload_hash) => { + let should_prepare_execution_payload = Feature::AlwaysPrepareExecutionPayload.is_enabled() + || self.attestation_agg_pool.has_registered_validators_proposing_in_slots(slot..=slot).await; + + if !should_prepare_execution_payload { + continue; + } + let slot_head = self.safe_slot_head(slot).await; if let Some(slot_head) = slot_head { From ec17aae9e6fe8d547b510265679e7ca99f92c6d0 Mon Sep 17 00:00:00 2001 From: Tumas Date: Tue, 18 Jun 2024 15:35:24 +0300 Subject: [PATCH 2/2] Fix issues with thread count: - Update Rayon to the latest version - Replaced Rayon with std::thread in state transitions to reduce dependency on the Rayon library - Properly disable HiGHS parallelism - Add DebugAttestationPacker feature for more granular AttestationPacker debugging - Update AttestationPacker metrics in Grafana - Add thread count metrics --- Cargo.lock | 14 +- Cargo.toml | 3 +- features/src/lib.rs | 7 +- grandine-snapshot-tests | 2 +- metrics/Cargo.toml | 1 + metrics/src/beaconchain.rs | 6 + metrics/src/server.rs | 1 + .../attestation_packer.rs | 3 +- .../src/attestation_agg_pool/tasks.rs | 36 +- prometheus_metrics/dashboards/overview.json | 261 +++++++++++-- prometheus_metrics/dashboards/pools.json | 368 +++++++++--------- prometheus_metrics/src/metrics.rs | 35 +- .../src/altair/state_transition.rs | 20 +- .../src/bellatrix/state_transition.rs | 20 +- .../src/capella/state_transition.rs | 20 +- .../src/deneb/state_transition.rs | 20 +- .../src/phase0/state_transition.rs | 20 +- 17 files changed, 574 insertions(+), 263 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bcaefbaa..454f20b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4851,6 +4851,7 @@ dependencies = [ "http_api_utils", "jemalloc-ctl", "log", + "num_threads", "p2p", "prometheus", "prometheus-client", @@ -5168,6 +5169,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "object" version = "0.32.2" @@ -6123,9 +6133,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", diff --git a/Cargo.toml b/Cargo.toml index 3a3e4e58..377dddc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -344,6 +344,7 @@ num-bigint = '0.4.4' num-integer = '0.1.45' num-traits = '0.2.17' num_cpus = '1.16.0' +num_threads = '0.1.7' once_cell = '1.19.0' openssl = '0.10.63' parking_lot = '0.12.1' @@ -362,7 +363,7 @@ quickcheck = '1.0.3' quickcheck_macros = '1.0.0' quote = '1.0.35' rand = '0.8.5' -rayon = '1.8.1' +rayon = '1.10.0' rc-box = '1.2.0' refinery = { version = '0.8.12', features = ['rusqlite']} regex = '1.10.3' diff --git a/features/src/lib.rs b/features/src/lib.rs index cabf0214..edd9d05d 100644 --- a/features/src/lib.rs +++ b/features/src/lib.rs @@ -8,7 +8,7 @@ use core::{ }; use enum_iterator::Sequence; -use log::info; +use log::{info, warn}; use parse_display::{Display, FromStr}; use serde::{Deserialize, Serialize}; @@ -41,6 +41,7 @@ pub enum Feature { AlwaysPrepackAttestations, AlwaysPrepareExecutionPayload, CacheTargetStates, + DebugAttestationPacker, DebugEth1, DebugP2p, IgnoreAttestationsForUnknownBlocks, @@ -98,6 +99,10 @@ impl Feature { // Maybe `log::kv` will be stable someday. Or we could implement it ourselves. info!("[{self}] {message}"); } + + pub fn warn(self, message: impl Display) { + warn!("[{self}] {message}"); + } } #[macro_export] diff --git a/grandine-snapshot-tests b/grandine-snapshot-tests index ca7f8884..cffcdf7a 160000 --- a/grandine-snapshot-tests +++ b/grandine-snapshot-tests @@ -1 +1 @@ -Subproject commit ca7f888404c36a5ad816bf93d76a0e5fb4e6249f +Subproject commit cffcdf7a7734fc4b70f5942e88cc89d054e2c85d diff --git a/metrics/Cargo.toml b/metrics/Cargo.toml index 40806959..69be48e3 100644 --- a/metrics/Cargo.toml +++ b/metrics/Cargo.toml @@ -22,6 +22,7 @@ helper_functions = { workspace = true } http_api_utils = { workspace = true } jemalloc-ctl = { workspace = true } log = { workspace = true } +num_threads = { workspace = true } p2p = { workspace = true } prometheus = { workspace = true } # `prometheus-client` is only needed for libp2p metrics. diff --git a/metrics/src/beaconchain.rs b/metrics/src/beaconchain.rs index 25f53e14..4d2a3cc0 100644 --- a/metrics/src/beaconchain.rs +++ b/metrics/src/beaconchain.rs @@ -82,6 +82,8 @@ pub struct ProcessMetrics { pub client_build: i64, pub sync_eth2_fallback_configured: bool, pub sync_eth2_fallback_connected: bool, + #[serde(skip)] + pub thread_count: usize, } impl ProcessMetrics { @@ -123,6 +125,9 @@ impl ProcessMetrics { // Grandine does not support Eth2 sync fallbacks. sync_eth2_fallback_configured: false, sync_eth2_fallback_connected: false, + thread_count: num_threads::num_threads() + .map(Into::into) + .unwrap_or_default(), } } } @@ -393,6 +398,7 @@ mod tests { client_build: 12, sync_eth2_fallback_configured: false, sync_eth2_fallback_connected: false, + thread_count: 32, }; [ diff --git a/metrics/src/server.rs b/metrics/src/server.rs index b7911b49..a32a0e8f 100644 --- a/metrics/src/server.rs +++ b/metrics/src/server.rs @@ -269,6 +269,7 @@ async fn scrape_system_stats( let process_metrics = ProcessMetrics::get(); + metrics.set_grandine_thread_count(process_metrics.thread_count); metrics.set_total_cpu_seconds(process_metrics.cpu_process_seconds_total); Ok(()) diff --git a/operation_pools/src/attestation_agg_pool/attestation_packer.rs b/operation_pools/src/attestation_agg_pool/attestation_packer.rs index 2fbf7e33..7ffdd482 100644 --- a/operation_pools/src/attestation_agg_pool/attestation_packer.rs +++ b/operation_pools/src/attestation_agg_pool/attestation_packer.rs @@ -490,10 +490,11 @@ impl AttestationPacker

{ let mut selected_aggregates = Vec::new(); // Here parallelization is unnecessary, since integer programming is called for each different attestation data - problem = problem.set_parallel(HighsParallelType::Off); + problem = problem.set_parallel(HighsParallelType::Off).set_threads(1); problem = problem .set_time_limit(self.time_until_deadline_in_seconds()? * TIME_FRACTION_FOR_SUBPROBLEM); + let solution = problem.solve()?; for (i, is_selected) in is_aggregate_selected.iter().enumerate() { diff --git a/operation_pools/src/attestation_agg_pool/tasks.rs b/operation_pools/src/attestation_agg_pool/tasks.rs index 4fdd31da..26a67ea2 100644 --- a/operation_pools/src/attestation_agg_pool/tasks.rs +++ b/operation_pools/src/attestation_agg_pool/tasks.rs @@ -1,9 +1,10 @@ use core::time::Duration; -use std::sync::Arc; +use std::{sync::Arc, time::Instant}; use anyhow::Result; use bls::PublicKeyBytes; use eth1_api::ApiController; +use features::Feature::DebugAttestationPacker; use fork_choice_control::Wait; use helper_functions::accessors; use prometheus_metrics::Metrics; @@ -40,11 +41,18 @@ impl PoolTask for BestProposableAttestationsTask { } = self; let attestations = pool.best_proposable_attestations(beacon_state.slot()).await; + let slot = controller.slot(); if !attestations.is_empty() { + features::log!( + DebugAttestationPacker, + "optimal attestations present for slot: {slot}" + ); return Ok(attestations); } + DebugAttestationPacker.warn("no optimal attestations for slot: {slot}"); + let attestation_packer = AttestationPacker::new( controller.chain_config().clone_arc(), controller.head_block_root().value, @@ -98,6 +106,7 @@ impl PoolTask for PackProposableAttestationsTask { } = self; let beacon_state = controller.preprocessed_state_at_next_slot()?; + let slot = controller.slot() + 1; let mut attestation_packer = AttestationPacker::new( controller.chain_config().clone_arc(), @@ -107,19 +116,28 @@ impl PoolTask for PackProposableAttestationsTask { )?; let mut is_empty = true; + let mut iteration: u32 = 0; loop { let PackOutcome { attestations, deadline_reached, } = { - let _timer = metrics.as_ref().map(|metrics| { - metrics - .att_pool_pack_proposable_attestation_task_times - .start_timer() - }); + let timer = Instant::now(); + + let outcome = + pack_attestations_optimally(&attestation_packer, &pool, &beacon_state).await; - pack_attestations_optimally(&attestation_packer, &pool, &beacon_state).await + features::log!( + DebugAttestationPacker, + "pack outcome for slot: {slot}, attestations: {}, iteration: {iteration}, deadline_reached: {}, time: {:?}", + outcome.attestations.len(), + outcome.deadline_reached, + timer.elapsed() + ); + + iteration += 1; + outcome }; if is_empty || !deadline_reached { @@ -129,6 +147,10 @@ impl PoolTask for PackProposableAttestationsTask { } if deadline_reached { + if let Some(metrics) = metrics.as_ref() { + metrics.set_attestation_packer_iteration_count(iteration.saturating_sub(1)); + } + break; } diff --git a/prometheus_metrics/dashboards/overview.json b/prometheus_metrics/dashboards/overview.json index 60363471..e5b912e2 100644 --- a/prometheus_metrics/dashboards/overview.json +++ b/prometheus_metrics/dashboards/overview.json @@ -410,6 +410,7 @@ "y": 9 }, "id": 11, + "interval": "1s", "options": { "legend": { "calcs": [], @@ -692,13 +693,206 @@ "title": "System Used memory", "type": "timeseries" }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 24, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 40, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "GRANDINE_TOTAL_CPU_PERCENTAGE{instance=~\"$Instance\"}", + "instant": false, + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "exemplar": false, + "expr": "rate(process_cpu_seconds_total{instance=~\"$Instance\"}[$__rate_interval]) * 100", + "hide": false, + "instant": false, + "legendFormat": "process_cpu_seconds_total {{label_name}} {{instance}}", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "rate(GRANDINE_TOTAL_CPU_SECONDS{instance=~\"$Instance\"}[$__rate_interval]) * 100", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "GRANDINE_TOTAL_CPU_SECONDS {{instance}}", + "range": true, + "refId": "C" + } + ], + "title": "Grandine vs Lighthouse CPU ", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "GRANDINE_THREAD_COUNT{instance=~\"$Instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Grandine Thread Count", + "type": "timeseries" + }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 41 }, "id": 16, "panels": [ @@ -759,7 +953,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 34 }, "id": 18, "options": { @@ -842,7 +1036,7 @@ "h": 8, "w": 8, "x": 12, - "y": 26 + "y": 34 }, "id": 19, "options": { @@ -895,7 +1089,7 @@ "h": 8, "w": 4, "x": 20, - "y": 26 + "y": 34 }, "id": 17, "options": { @@ -935,7 +1129,7 @@ "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 42 }, "id": 33, "panels": [ @@ -997,7 +1191,7 @@ "h": 8, "w": 12, "x": 0, - "y": 27 + "y": 35 }, "id": 34, "options": { @@ -1082,7 +1276,7 @@ "h": 8, "w": 12, "x": 12, - "y": 27 + "y": 35 }, "id": 35, "options": { @@ -1167,7 +1361,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 43 }, "id": 36, "options": { @@ -1252,7 +1446,7 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 43 }, "id": 37, "options": { @@ -1337,7 +1531,7 @@ "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 51 }, "id": 38, "options": { @@ -1422,7 +1616,7 @@ "h": 8, "w": 12, "x": 12, - "y": 43 + "y": 51 }, "id": 39, "options": { @@ -1459,7 +1653,7 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 43 }, "id": 12, "panels": [], @@ -1508,7 +1702,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1523,7 +1718,7 @@ "h": 21, "w": 24, "x": 0, - "y": 28 + "y": 44 }, "id": 13, "options": { @@ -1557,7 +1752,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 65 }, "id": 9, "panels": [ @@ -1618,7 +1813,7 @@ "h": 8, "w": 12, "x": 0, - "y": 52 + "y": 60 }, "id": 6, "options": { @@ -1702,7 +1897,7 @@ "h": 8, "w": 12, "x": 12, - "y": 52 + "y": 60 }, "id": 7, "options": { @@ -1739,7 +1934,7 @@ "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 66 }, "id": 14, "panels": [ @@ -1800,7 +1995,7 @@ "h": 8, "w": 24, "x": 0, - "y": 30 + "y": 38 }, "id": 15, "options": { @@ -1837,7 +2032,7 @@ "h": 1, "w": 24, "x": 0, - "y": 51 + "y": 67 }, "id": 25, "panels": [], @@ -1866,7 +2061,7 @@ "h": 8, "w": 12, "x": 0, - "y": 52 + "y": 68 }, "id": 27, "options": { @@ -1918,7 +2113,7 @@ "h": 8, "w": 12, "x": 12, - "y": 52 + "y": 68 }, "id": 26, "options": { @@ -1970,7 +2165,7 @@ "h": 8, "w": 12, "x": 0, - "y": 60 + "y": 76 }, "id": 23, "options": { @@ -2022,7 +2217,7 @@ "h": 8, "w": 12, "x": 12, - "y": 60 + "y": 76 }, "id": 24, "options": { @@ -2109,7 +2304,7 @@ "h": 8, "w": 6, "x": 0, - "y": 68 + "y": 84 }, "id": 28, "options": { @@ -2193,7 +2388,7 @@ "h": 8, "w": 6, "x": 6, - "y": 68 + "y": 84 }, "id": 29, "options": { @@ -2243,7 +2438,7 @@ "h": 8, "w": 12, "x": 12, - "y": 68 + "y": 84 }, "id": 32, "options": { @@ -2330,7 +2525,7 @@ "h": 8, "w": 12, "x": 0, - "y": 76 + "y": 92 }, "id": 30, "options": { @@ -2414,7 +2609,7 @@ "h": 8, "w": 12, "x": 12, - "y": 76 + "y": 92 }, "id": 31, "options": { @@ -2452,10 +2647,10 @@ "current": { "selected": true, "text": [ - "0.0.0.0:5061" + "0.0.0.0:5056" ], "value": [ - "0.0.0.0:5061" + "0.0.0.0:5056" ] }, "definition": "label_values(instance)", @@ -2477,13 +2672,13 @@ ] }, "time": { - "from": "now-15m", + "from": "now-5m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Overview", "uid": "eb977235-e486-4662-9e99-a2631b9eddca", - "version": 8, + "version": 12, "weekStart": "" } \ No newline at end of file diff --git a/prometheus_metrics/dashboards/pools.json b/prometheus_metrics/dashboards/pools.json index 11cc05b9..1f616fe2 100644 --- a/prometheus_metrics/dashboards/pools.json +++ b/prometheus_metrics/dashboards/pools.json @@ -23,7 +23,7 @@ "liveNow": false, "panels": [ { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -31,184 +31,184 @@ "y": 0 }, "id": 2, - "panels": [ - { - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-orange", - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 + "panels": [], + "title": "Attestation Agg pool", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 39, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "editorMode": "code", - "expr": "rate(ATT_POOL_PACK_PROPOSABLE_ATTESTATION_TASK_TIMES_sum{instance=~\"$Instance\"}[$__rate_interval]) / rate(ATT_POOL_PACK_PROPOSABLE_ATTESTATION_TASK_TIMES_count{instance=~\"$Instance\"}[$__rate_interval])\n", - "instant": false, - "range": true, - "refId": "A" - } - ], - "title": "Pack proposal attestation task", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "super-light-orange", - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "unit": "s" - }, - "overrides": [] + { + "color": "red", + "value": 80 + } + ] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 14, + "interval": "12s", + "options": { + "barRadius": 0, + "barWidth": 1, + "fullHighlight": true, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "always", + "stacking": "none", + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": -30, + "xTickLabelSpacing": 100 + }, + "targets": [ + { + "editorMode": "code", + "expr": "ATT_POOL_PACK_ITERATIONS{instance=~\"$Instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Attestation Packer iterations per slot", + "type": "barchart" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-orange", + "mode": "palette-classic" }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "editorMode": "code", - "expr": "rate(ATT_POOL_INSERT_ATTESTATION_TASK_TIMES_sum{instance=~\"$Instance\"}[$__rate_interval]) / rate(ATT_POOL_INSERT_ATTESTATION_TASK_TIMES_count{instance=~\"$Instance\"}[$__rate_interval])\n", - "instant": false, - "range": true, - "refId": "A" - } - ], - "title": "Insert attestation task", - "type": "timeseries" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "rate(ATT_POOL_INSERT_ATTESTATION_TASK_TIMES_sum{instance=~\"$Instance\"}[$__rate_interval]) / rate(ATT_POOL_INSERT_ATTESTATION_TASK_TIMES_count{instance=~\"$Instance\"}[$__rate_interval])\n", + "instant": false, + "range": true, + "refId": "A" } ], - "title": "Attestation Agg pool", - "type": "row" + "title": "Insert attestation task", + "type": "timeseries" }, { "collapsed": false, @@ -216,7 +216,7 @@ "h": 1, "w": 24, "x": 0, - "y": 1 + "y": 10 }, "id": 3, "panels": [], @@ -283,7 +283,7 @@ "h": 8, "w": 12, "x": 0, - "y": 2 + "y": 11 }, "id": 6, "options": { @@ -370,7 +370,7 @@ "h": 8, "w": 12, "x": 12, - "y": 2 + "y": 11 }, "id": 7, "options": { @@ -457,7 +457,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 19 }, "id": 8, "options": { @@ -544,7 +544,7 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 19 }, "id": 9, "options": { @@ -630,7 +630,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 27 }, "id": 10, "options": { @@ -715,7 +715,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 27 }, "id": 13, "options": { @@ -753,7 +753,7 @@ "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 35 }, "id": 4, "panels": [ @@ -800,8 +800,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -817,7 +816,7 @@ "h": 8, "w": 12, "x": 0, - "y": 3 + "y": 11 }, "id": 11, "options": { @@ -887,8 +886,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -904,7 +902,7 @@ "h": 8, "w": 12, "x": 12, - "y": 3 + "y": 11 }, "id": 12, "options": { @@ -946,10 +944,10 @@ "current": { "selected": true, "text": [ - "All" + "0.0.0.0:5054" ], "value": [ - "$__all" + "0.0.0.0:5054" ] }, "definition": "label_values(instance)", @@ -971,13 +969,13 @@ ] }, "time": { - "from": "now-5m", + "from": "now-15m", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Pools", "uid": "ed4add54-90fb-48a8-b185-83e25b1c2e46", - "version": 7, + "version": 11, "weekStart": "" } \ No newline at end of file diff --git a/prometheus_metrics/src/metrics.rs b/prometheus_metrics/src/metrics.rs index 45fd2435..cc0efab3 100644 --- a/prometheus_metrics/src/metrics.rs +++ b/prometheus_metrics/src/metrics.rs @@ -30,6 +30,7 @@ pub struct Metrics { system_total_memory: IntGauge, total_cpu_percentage: Gauge, total_cpu_seconds: IntGauge, + grandine_thread_count: IntGauge, // Collection Lengths collection_lengths: IntGaugeVec, @@ -99,7 +100,7 @@ pub struct Metrics { pub prepare_voluntary_exits_times: Histogram, // Pools - pub att_pool_pack_proposable_attestation_task_times: Histogram, + att_pool_pack_iterations: Gauge, pub att_pool_insert_attestation_task_times: Histogram, pub sync_pool_add_own_contribution_times: Histogram, @@ -204,10 +205,10 @@ impl Metrics { "Grandine CPU load usage measured in seconds" )?, + grandine_thread_count: IntGauge::new("GRANDINE_THREAD_COUNT", "Grandine task count")?, system_used_memory: IntGauge::new("SYSTEM_USED_MEMORY", "Node memory usage")?, system_total_memory: IntGauge::new("SYSTEM_TOTAL_MEMORY", "Total node mmeory")?, - // Collection Lengths collection_lengths: IntGaugeVec::new( opts!("COLLECTION_LENGTHS", "Number of items in each collection"), @@ -306,14 +307,6 @@ impl Metrics { "Attestation verifier active task count", )?, - // attestation_verifier_active_task_count: IntGaugeVec::new( - // opts!( - // "ATTESTATION_VERIFIER_ACTIVE_TASK_COUNT", - // "Attestation verifier active task count", - // ), - // &["type"], - // )?, - attestation_verifier_process_attestation_batch_times: Histogram::with_opts( histogram_opts!( "ATTESTATION_VERIFIER_PROCESS_ATTESTATION_BATCH_TIMES", @@ -443,10 +436,10 @@ impl Metrics { ))?, // Pools - att_pool_pack_proposable_attestation_task_times: Histogram::with_opts(histogram_opts!( - "ATT_POOL_PACK_PROPOSABLE_ATTESTATION_TASK_TIMES", - "Attestation agg pool packing proposable attestation task times", - ))?, + att_pool_pack_iterations: Gauge::new( + "ATT_POOL_PACK_ITERATIONS", + "Attestation packer iteration count per slog", + )?, att_pool_insert_attestation_task_times: Histogram::with_opts(histogram_opts!( "ATT_POOL_INSERT_ATTESTATION_TASK_TIMES", @@ -740,6 +733,7 @@ impl Metrics { default_registry.register(Box::new(self.system_total_memory.clone()))?; default_registry.register(Box::new(self.total_cpu_percentage.clone()))?; default_registry.register(Box::new(self.total_cpu_seconds.clone()))?; + default_registry.register(Box::new(self.grandine_thread_count.clone()))?; default_registry.register(Box::new(self.collection_lengths.clone()))?; default_registry.register(Box::new(self.http_api_response_times.clone()))?; default_registry.register(Box::new(self.validator_api_response_times.clone()))?; @@ -798,9 +792,7 @@ impl Metrics { default_registry.register(Box::new(self.prepare_attester_slashings_times.clone()))?; default_registry.register(Box::new(self.prepare_proposer_slashings_times.clone()))?; default_registry.register(Box::new(self.prepare_voluntary_exits_times.clone()))?; - default_registry.register(Box::new( - self.att_pool_pack_proposable_attestation_task_times.clone(), - ))?; + default_registry.register(Box::new(self.att_pool_pack_iterations.clone()))?; default_registry.register(Box::new( self.att_pool_insert_attestation_task_times.clone(), ))?; @@ -922,6 +914,10 @@ impl Metrics { self.system_total_memory.set(total_memory as i64) } + pub fn set_grandine_thread_count(&self, thread_count: usize) { + self.grandine_thread_count.set(thread_count as i64) + } + // Collection Lengths pub fn set_collection_length( &self, @@ -1100,4 +1096,9 @@ impl Metrics { ) .set(delay.as_secs_f64()) } + + // Pool + pub fn set_attestation_packer_iteration_count(&self, iterations: u32) { + self.att_pool_pack_iterations.set(iterations as f64) + } } diff --git a/transition_functions/src/altair/state_transition.rs b/transition_functions/src/altair/state_transition.rs index 63782bee..f8238bb2 100644 --- a/transition_functions/src/altair/state_transition.rs +++ b/transition_functions/src/altair/state_transition.rs @@ -1,6 +1,6 @@ use core::ops::Not as _; -use anyhow::{Error as AnyhowError, Result}; +use anyhow::{anyhow, Error as AnyhowError, Result}; use helper_functions::{ accessors, error::SignatureKind, @@ -62,8 +62,22 @@ pub fn state_transition( }; if let Some(verify_signatures) = verify_signatures { - let (signature_result, block_result) = rayon::join(verify_signatures, process_block); - signature_result.and(block_result) + std::thread::scope(|scope| { + let verify_signatures = scope.spawn(verify_signatures); + let process_block = scope.spawn(process_block); + + let signature_result = verify_signatures + .join() + .map_err(|_| anyhow!("failed to verify signatures")) + .and_then(|result| result); + + let block_result = process_block + .join() + .map_err(|_| anyhow!("failed to process block")) + .and_then(|result| result); + + signature_result.and(block_result) + }) } else { process_block() } diff --git a/transition_functions/src/bellatrix/state_transition.rs b/transition_functions/src/bellatrix/state_transition.rs index b201c511..edf769cb 100644 --- a/transition_functions/src/bellatrix/state_transition.rs +++ b/transition_functions/src/bellatrix/state_transition.rs @@ -1,6 +1,6 @@ use core::ops::Not as _; -use anyhow::{Error as AnyhowError, Result}; +use anyhow::{anyhow, Error as AnyhowError, Result}; use execution_engine::ExecutionEngine; use helper_functions::{ accessors, @@ -67,8 +67,22 @@ pub fn state_transition( }; if let Some(verify_signatures) = verify_signatures { - let (signature_result, block_result) = rayon::join(verify_signatures, process_block); - signature_result.and(block_result) + std::thread::scope(|scope| { + let verify_signatures = scope.spawn(verify_signatures); + let process_block = scope.spawn(process_block); + + let signature_result = verify_signatures + .join() + .map_err(|_| anyhow!("failed to verify signatures")) + .and_then(|result| result); + + let block_result = process_block + .join() + .map_err(|_| anyhow!("failed to process block")) + .and_then(|result| result); + + signature_result.and(block_result) + }) } else { process_block() } diff --git a/transition_functions/src/capella/state_transition.rs b/transition_functions/src/capella/state_transition.rs index aeb19780..f426f1a0 100644 --- a/transition_functions/src/capella/state_transition.rs +++ b/transition_functions/src/capella/state_transition.rs @@ -1,6 +1,6 @@ use core::ops::Not as _; -use anyhow::{Error as AnyhowError, Result}; +use anyhow::{anyhow, Error as AnyhowError, Result}; use execution_engine::ExecutionEngine; use helper_functions::{ accessors, @@ -67,8 +67,22 @@ pub fn state_transition( }; if let Some(verify_signatures) = verify_signatures { - let (signature_result, block_result) = rayon::join(verify_signatures, process_block); - signature_result.and(block_result) + std::thread::scope(|scope| { + let verify_signatures = scope.spawn(verify_signatures); + let process_block = scope.spawn(process_block); + + let signature_result = verify_signatures + .join() + .map_err(|_| anyhow!("failed to verify signatures")) + .and_then(|result| result); + + let block_result = process_block + .join() + .map_err(|_| anyhow!("failed to process block")) + .and_then(|result| result); + + signature_result.and(block_result) + }) } else { process_block() } diff --git a/transition_functions/src/deneb/state_transition.rs b/transition_functions/src/deneb/state_transition.rs index a8414066..bd747e5a 100644 --- a/transition_functions/src/deneb/state_transition.rs +++ b/transition_functions/src/deneb/state_transition.rs @@ -1,6 +1,6 @@ use core::ops::Not as _; -use anyhow::{Error as AnyhowError, Result}; +use anyhow::{anyhow, Error as AnyhowError, Result}; use execution_engine::ExecutionEngine; use helper_functions::{ accessors, @@ -67,8 +67,22 @@ pub fn state_transition( }; if let Some(verify_signatures) = verify_signatures { - let (signature_result, block_result) = rayon::join(verify_signatures, process_block); - signature_result.and(block_result) + std::thread::scope(|scope| { + let verify_signatures = scope.spawn(verify_signatures); + let process_block = scope.spawn(process_block); + + let signature_result = verify_signatures + .join() + .map_err(|_| anyhow!("failed to verify signatures")) + .and_then(|result| result); + + let block_result = process_block + .join() + .map_err(|_| anyhow!("failed to process block")) + .and_then(|result| result); + + signature_result.and(block_result) + }) } else { process_block() } diff --git a/transition_functions/src/phase0/state_transition.rs b/transition_functions/src/phase0/state_transition.rs index 843897b6..a8870189 100644 --- a/transition_functions/src/phase0/state_transition.rs +++ b/transition_functions/src/phase0/state_transition.rs @@ -1,6 +1,6 @@ use core::ops::Not as _; -use anyhow::{Error as AnyhowError, Result}; +use anyhow::{anyhow, Error as AnyhowError, Result}; use helper_functions::{ accessors, error::SignatureKind, @@ -64,8 +64,22 @@ pub fn state_transition( }; if let Some(verify_signatures) = verify_signatures { - let (signature_result, block_result) = rayon::join(verify_signatures, process_block); - signature_result.and(block_result) + std::thread::scope(|scope| { + let verify_signatures = scope.spawn(verify_signatures); + let process_block = scope.spawn(process_block); + + let signature_result = verify_signatures + .join() + .map_err(|_| anyhow!("failed to verify signatures")) + .and_then(|result| result); + + let block_result = process_block + .join() + .map_err(|_| anyhow!("failed to process block")) + .and_then(|result| result); + + signature_result.and(block_result) + }) } else { process_block() }