From a486f3c7a61f86e411beb68364fe6c6758fdeda8 Mon Sep 17 00:00:00 2001 From: gr0vity Date: Sun, 7 Jul 2024 13:01:34 +0200 Subject: [PATCH] Squash merge PR #4665: Bounded cementing --- nano/core_test/confirming_set.cpp | 22 +- nano/core_test/ledger.cpp | 289 ++++++++++++++++-------- nano/lib/stats_enums.hpp | 4 + nano/lib/thread_roles.cpp | 2 +- nano/lib/thread_roles.hpp | 2 +- nano/node/confirming_set.cpp | 90 +++++--- nano/node/confirming_set.hpp | 35 +-- nano/node/fwd.hpp | 1 + nano/node/node.cpp | 2 +- nano/node/nodeconfig.hpp | 2 + nano/secure/ledger.cpp | 39 +++- nano/secure/ledger.hpp | 18 +- nano/slow_test/node.cpp | 3 +- nano/test_common/CMakeLists.txt | 4 +- nano/test_common/ledger.cpp | 106 --------- nano/test_common/ledger.hpp | 47 ---- nano/test_common/ledger_context.cpp | 328 ++++++++++++++++++++++++++++ nano/test_common/ledger_context.hpp | 42 ++++ 18 files changed, 710 insertions(+), 326 deletions(-) delete mode 100644 nano/test_common/ledger.cpp delete mode 100644 nano/test_common/ledger.hpp create mode 100644 nano/test_common/ledger_context.cpp create mode 100644 nano/test_common/ledger_context.hpp diff --git a/nano/core_test/confirming_set.cpp b/nano/core_test/confirming_set.cpp index f2fe621065..31ca879511 100644 --- a/nano/core_test/confirming_set.cpp +++ b/nano/core_test/confirming_set.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include @@ -18,14 +18,16 @@ using namespace std::chrono_literals; TEST (confirming_set, construction) { - auto ctx = nano::test::context::ledger_empty (); - nano::confirming_set confirming_set (ctx.ledger (), ctx.stats ()); + auto ctx = nano::test::ledger_empty (); + nano::confirming_set_config config{}; + nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats () }; } TEST (confirming_set, add_exists) { - auto ctx = nano::test::context::ledger_send_receive (); - nano::confirming_set confirming_set (ctx.ledger (), ctx.stats ()); + auto ctx = nano::test::ledger_send_receive (); + nano::confirming_set_config config{}; + nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats () }; auto send = ctx.blocks ()[0]; confirming_set.add (send->hash ()); ASSERT_TRUE (confirming_set.exists (send->hash ())); @@ -33,8 +35,9 @@ TEST (confirming_set, add_exists) TEST (confirming_set, process_one) { - auto ctx = nano::test::context::ledger_send_receive (); - nano::confirming_set confirming_set (ctx.ledger (), ctx.stats ()); + auto ctx = nano::test::ledger_send_receive (); + nano::confirming_set_config config{}; + nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats () }; std::atomic count = 0; std::mutex mutex; std::condition_variable condition; @@ -49,8 +52,9 @@ TEST (confirming_set, process_one) TEST (confirming_set, process_multiple) { - auto ctx = nano::test::context::ledger_send_receive (); - nano::confirming_set confirming_set (ctx.ledger (), ctx.stats ()); + auto ctx = nano::test::ledger_send_receive (); + nano::confirming_set_config config{}; + nano::confirming_set confirming_set{ config, ctx.ledger (), ctx.stats () }; std::atomic count = 0; std::mutex mutex; std::condition_variable condition; diff --git a/nano/core_test/ledger.cpp b/nano/core_test/ledger.cpp index 850ddab934..1156ad11a8 100644 --- a/nano/core_test/ledger.cpp +++ b/nano/core_test/ledger.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -32,13 +32,13 @@ TEST (ledger, store_error) // Don't test this in rocksdb mode GTEST_SKIP (); } - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); } // Ledger can be initialized and returns a basic query for an empty account TEST (ledger, empty) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_read (); @@ -50,7 +50,7 @@ TEST (ledger, empty) // Ledger can be initialized and returns a basic query for an empty account TEST (ledger, confirmed_unconfirmed_view) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & unconfirmed = ledger; auto & confirmed = ledger.confirmed; @@ -59,7 +59,7 @@ TEST (ledger, confirmed_unconfirmed_view) // Genesis account should have the max balance on empty initialization TEST (ledger, genesis_balance) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -80,7 +80,7 @@ TEST (ledger, genesis_balance) TEST (ledger, process_modifies_sideband) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto & pool = ctx.pool (); @@ -102,7 +102,7 @@ TEST (ledger, process_modifies_sideband) // Create a send block and publish it. TEST (ledger, process_send) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -205,7 +205,7 @@ TEST (ledger, process_send) TEST (ledger, process_receive) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -288,7 +288,7 @@ TEST (ledger, process_receive) TEST (ledger, rollback_receiver) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -337,7 +337,7 @@ TEST (ledger, rollback_receiver) TEST (ledger, rollback_representation) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -427,7 +427,7 @@ TEST (ledger, rollback_representation) TEST (ledger, receive_rollback) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -455,7 +455,7 @@ TEST (ledger, receive_rollback) TEST (ledger, process_duplicate) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -489,7 +489,7 @@ TEST (ledger, process_duplicate) TEST (ledger, representative_genesis) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -500,14 +500,14 @@ TEST (ledger, representative_genesis) TEST (ledger, weight) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); ASSERT_EQ (nano::dev::constants.genesis_amount, ledger.weight (nano::dev::genesis_key.pub)); } TEST (ledger, representative_change) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -544,7 +544,7 @@ TEST (ledger, representative_change) TEST (ledger, send_fork) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); nano::keypair key2; @@ -576,7 +576,7 @@ TEST (ledger, send_fork) TEST (ledger, receive_fork) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); nano::keypair key2; @@ -633,7 +633,7 @@ TEST (ledger, receive_fork) TEST (ledger, open_fork) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); nano::keypair key2; @@ -734,7 +734,7 @@ TEST (ledger, rep_cache_min_weight) TEST (ledger, representation) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto & rep_weights = ledger.cache.rep_weights; @@ -907,7 +907,7 @@ TEST (ledger, double_open) TEST (ledger, double_receive) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1244,7 +1244,7 @@ TEST (ledger, successor) TEST (ledger, fail_change_old) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1266,7 +1266,7 @@ TEST (ledger, fail_change_old) TEST (ledger, fail_change_gap_previous) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1286,7 +1286,7 @@ TEST (ledger, fail_change_gap_previous) TEST (ledger, fail_state_bad_signature) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1308,7 +1308,7 @@ TEST (ledger, fail_state_bad_signature) TEST (ledger, fail_epoch_bad_signature) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1334,7 +1334,7 @@ TEST (ledger, fail_epoch_bad_signature) TEST (ledger, fail_change_bad_signature) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1354,7 +1354,7 @@ TEST (ledger, fail_change_bad_signature) TEST (ledger, fail_change_fork) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1384,7 +1384,7 @@ TEST (ledger, fail_change_fork) TEST (ledger, fail_send_old) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1407,7 +1407,7 @@ TEST (ledger, fail_send_old) TEST (ledger, fail_send_gap_previous) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1428,7 +1428,7 @@ TEST (ledger, fail_send_gap_previous) TEST (ledger, fail_send_bad_signature) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1449,7 +1449,7 @@ TEST (ledger, fail_send_bad_signature) TEST (ledger, fail_send_negative_spend) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1479,7 +1479,7 @@ TEST (ledger, fail_send_negative_spend) TEST (ledger, fail_send_fork) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1509,7 +1509,7 @@ TEST (ledger, fail_send_fork) TEST (ledger, fail_open_old) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1539,7 +1539,7 @@ TEST (ledger, fail_open_old) TEST (ledger, fail_open_gap_source) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1560,7 +1560,7 @@ TEST (ledger, fail_open_gap_source) TEST (ledger, fail_open_bad_signature) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1590,7 +1590,7 @@ TEST (ledger, fail_open_bad_signature) TEST (ledger, fail_open_fork_previous) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1638,7 +1638,7 @@ TEST (ledger, fail_open_fork_previous) TEST (ledger, fail_open_account_mismatch) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1669,7 +1669,7 @@ TEST (ledger, fail_open_account_mismatch) TEST (ledger, fail_receive_old) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1716,7 +1716,7 @@ TEST (ledger, fail_receive_old) TEST (ledger, fail_receive_gap_source) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1766,7 +1766,7 @@ TEST (ledger, fail_receive_gap_source) TEST (ledger, fail_receive_overreceive) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1806,7 +1806,7 @@ TEST (ledger, fail_receive_overreceive) TEST (ledger, fail_receive_bad_signature) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1856,7 +1856,7 @@ TEST (ledger, fail_receive_bad_signature) TEST (ledger, fail_receive_gap_previous_opened) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1906,7 +1906,7 @@ TEST (ledger, fail_receive_gap_previous_opened) TEST (ledger, fail_receive_gap_previous_unopened) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -1946,7 +1946,7 @@ TEST (ledger, fail_receive_gap_previous_unopened) TEST (ledger, fail_receive_fork_previous) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2007,7 +2007,7 @@ TEST (ledger, fail_receive_fork_previous) TEST (ledger, fail_receive_received_source) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2087,7 +2087,7 @@ TEST (ledger, fail_receive_received_source) TEST (ledger, latest_empty) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); nano::keypair key; @@ -2098,7 +2098,7 @@ TEST (ledger, latest_empty) TEST (ledger, latest_root) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2121,7 +2121,7 @@ TEST (ledger, latest_root) TEST (ledger, change_representative_move_representation) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); nano::keypair key1; @@ -2163,7 +2163,7 @@ TEST (ledger, change_representative_move_representation) TEST (ledger, send_open_receive_rollback) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2252,7 +2252,7 @@ TEST (ledger, send_open_receive_rollback) TEST (ledger, bootstrap_rep_weight) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); nano::keypair key2; @@ -2299,7 +2299,7 @@ TEST (ledger, bootstrap_rep_weight) TEST (ledger, block_destination_source) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2389,7 +2389,7 @@ TEST (ledger, block_destination_source) TEST (ledger, state_account) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2411,7 +2411,7 @@ TEST (ledger, state_account) TEST (ledger, state_send_receive) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2468,7 +2468,7 @@ TEST (ledger, state_send_receive) TEST (ledger, state_receive) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2516,7 +2516,7 @@ TEST (ledger, state_receive) TEST (ledger, state_rep_change) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2550,7 +2550,7 @@ TEST (ledger, state_rep_change) TEST (ledger, state_open) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2605,7 +2605,7 @@ TEST (ledger, state_open) // Make sure old block types can't be inserted after a state block. TEST (ledger, send_after_state_fail) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2636,7 +2636,7 @@ TEST (ledger, send_after_state_fail) // Make sure old block types can't be inserted after a state block. TEST (ledger, receive_after_state_fail) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2666,7 +2666,7 @@ TEST (ledger, receive_after_state_fail) // Make sure old block types can't be inserted after a state block. TEST (ledger, change_after_state_fail) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2696,7 +2696,7 @@ TEST (ledger, change_after_state_fail) TEST (ledger, state_unreceivable_fail) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2733,7 +2733,7 @@ TEST (ledger, state_unreceivable_fail) TEST (ledger, state_receive_bad_amount_fail) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2770,7 +2770,7 @@ TEST (ledger, state_receive_bad_amount_fail) TEST (ledger, state_no_link_amount_fail) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2803,7 +2803,7 @@ TEST (ledger, state_no_link_amount_fail) TEST (ledger, state_receive_wrong_account_fail) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2843,7 +2843,7 @@ TEST (ledger, state_receive_wrong_account_fail) TEST (ledger, state_open_state_fork) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2886,7 +2886,7 @@ TEST (ledger, state_open_state_fork) TEST (ledger, state_state_open_fork) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2930,7 +2930,7 @@ TEST (ledger, state_state_open_fork) TEST (ledger, state_open_previous_fail) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2963,7 +2963,7 @@ TEST (ledger, state_open_previous_fail) TEST (ledger, state_open_source_fail) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -2996,7 +2996,7 @@ TEST (ledger, state_open_source_fail) TEST (ledger, state_send_change) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3030,7 +3030,7 @@ TEST (ledger, state_send_change) TEST (ledger, state_receive_change) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3082,7 +3082,7 @@ TEST (ledger, state_receive_change) TEST (ledger, state_open_old) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3116,7 +3116,7 @@ TEST (ledger, state_open_old) TEST (ledger, state_receive_old) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3169,7 +3169,7 @@ TEST (ledger, state_receive_old) TEST (ledger, state_rollback_send) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3207,7 +3207,7 @@ TEST (ledger, state_rollback_send) TEST (ledger, state_rollback_receive) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3249,7 +3249,7 @@ TEST (ledger, state_rollback_receive) TEST (ledger, state_rollback_received_send) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3292,7 +3292,7 @@ TEST (ledger, state_rollback_received_send) TEST (ledger, state_rep_change_rollback) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3319,7 +3319,7 @@ TEST (ledger, state_rep_change_rollback) TEST (ledger, state_open_rollback) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3361,7 +3361,7 @@ TEST (ledger, state_open_rollback) TEST (ledger, state_send_change_rollback) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3389,7 +3389,7 @@ TEST (ledger, state_send_change_rollback) TEST (ledger, state_receive_change_rollback) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3428,7 +3428,7 @@ TEST (ledger, state_receive_change_rollback) TEST (ledger, epoch_blocks_v1_general) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3571,7 +3571,7 @@ TEST (ledger, epoch_blocks_v1_general) TEST (ledger, epoch_blocks_v2_general) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3736,7 +3736,7 @@ TEST (ledger, epoch_blocks_v2_general) TEST (ledger, epoch_blocks_receive_upgrade) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -3944,7 +3944,7 @@ TEST (ledger, epoch_blocks_receive_upgrade) TEST (ledger, epoch_blocks_fork) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -4471,7 +4471,7 @@ TEST (ledger, unchecked_receive) TEST (ledger, confirmation_height_not_updated) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -4544,7 +4544,7 @@ TEST (ledger, zero_rep) TEST (ledger, work_validation) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto & pool = ctx.pool (); @@ -4633,7 +4633,7 @@ TEST (ledger, work_validation) TEST (ledger, dependents_confirmed) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -4743,7 +4743,7 @@ TEST (ledger, dependents_confirmed_pruning) TEST (ledger, block_confirmed) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto transaction = ledger.tx_begin_write (); @@ -4770,7 +4770,7 @@ TEST (ledger, block_confirmed) TEST (ledger, cache) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto & stats = ctx.stats (); @@ -5487,7 +5487,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb) TEST (ledger, is_send_genesis) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto tx = store.tx_begin_read (); @@ -5496,7 +5496,7 @@ TEST (ledger, is_send_genesis) TEST (ledger, is_send_state) { - auto ctx = nano::test::context::ledger_send_receive (); + auto ctx = nano::test::ledger_send_receive (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto tx = store.tx_begin_read (); @@ -5506,7 +5506,7 @@ TEST (ledger, is_send_state) TEST (ledger, is_send_legacy) { - auto ctx = nano::test::context::ledger_send_receive_legacy (); + auto ctx = nano::test::ledger_send_receive_legacy (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto tx = store.tx_begin_read (); @@ -5516,24 +5516,119 @@ TEST (ledger, is_send_legacy) TEST (ledger, head_block) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); auto & ledger = ctx.ledger (); auto & store = ctx.store (); auto tx = ledger.tx_begin_read (); ASSERT_EQ (*nano::dev::genesis, *ledger.any.block_get (tx, ledger.any.account_head (tx, nano::dev::genesis_key.pub))); } +// Tests that a fairly complex ledger topology can be cemented in a single batch +TEST (ledger, cement_unbounded) +{ + auto ctx = nano::test::ledger_diamond (5); + auto & ledger = ctx.ledger (); + auto bottom = ctx.blocks ().back (); + auto confirmed = ledger.confirm (ledger.tx_begin_write (), bottom->hash ()); + ASSERT_TRUE (ledger.confirmed.block_exists (ledger.tx_begin_read (), bottom->hash ())); + // Check that all blocks got confirmed in a single call + ASSERT_TRUE (std::all_of (ctx.blocks ().begin (), ctx.blocks ().end (), [&] (auto const & block) { + return std::find_if (confirmed.begin (), confirmed.end (), [&] (auto const & block2) { + return block2->hash () == block->hash (); + }) + != confirmed.end (); + })); +} + +// Tests that bounded cementing works when recursion stack is large +TEST (ledger, cement_bounded) +{ + auto ctx = nano::test::ledger_single_chain (64); + auto & ledger = ctx.ledger (); + auto bottom = ctx.blocks ().back (); + + // This should only cement some of the dependencies + auto confirmed1 = ledger.confirm (ledger.tx_begin_write (), bottom->hash (), /* max cementing batch size */ 3); + ASSERT_FALSE (ledger.confirmed.block_exists (ledger.tx_begin_read (), bottom->hash ())); + ASSERT_EQ (confirmed1.size (), 3); + // Only topmost dependencies should get cemented during this call + ASSERT_TRUE (std::all_of (confirmed1.begin (), confirmed1.end (), [&] (auto const & block) { + return ledger.dependents_confirmed (ledger.tx_begin_read (), *block); + })); + + // This should cement a few more dependencies + auto confirmed2 = ledger.confirm (ledger.tx_begin_write (), bottom->hash (), /* max cementing batch size */ 16); + ASSERT_FALSE (ledger.confirmed.block_exists (ledger.tx_begin_read (), bottom->hash ())); + ASSERT_EQ (confirmed2.size (), 16); + // Only topmost dependencies should get cemented during this call + ASSERT_TRUE (std::all_of (confirmed2.begin (), confirmed2.end (), [&] (auto const & block) { + return ledger.dependents_confirmed (ledger.tx_begin_read (), *block); + })); + + // This should cement the remaining dependencies + auto confirmed3 = ledger.confirm (ledger.tx_begin_write (), bottom->hash (), /* max cementing batch size */ 64); + ASSERT_TRUE (ledger.confirmed.block_exists (ledger.tx_begin_read (), bottom->hash ())); + ASSERT_LE (confirmed3.size (), 64); + // Every block in the ledger should be cemented + ASSERT_TRUE (std::all_of (ctx.blocks ().begin (), ctx.blocks ().end (), [&] (auto const & block) { + return ledger.confirmed.block_exists (ledger.tx_begin_read (), block->hash ()); + })); +} + +// Tests that bounded cementing works when mumber of blocks is large but tree height is small (recursion stack is small) +TEST (ledger, cement_bounded_diamond) +{ + auto ctx = nano::test::ledger_diamond (4); + auto & ledger = ctx.ledger (); + auto bottom = ctx.blocks ().back (); + + // This should only cement some of the dependencies + auto confirmed1 = ledger.confirm (ledger.tx_begin_write (), bottom->hash (), /* max cementing batch size */ 3); + ASSERT_FALSE (ledger.confirmed.block_exists (ledger.tx_begin_read (), bottom->hash ())); + ASSERT_EQ (confirmed1.size (), 3); + // Only topmost dependencies should get cemented during this call + ASSERT_TRUE (std::all_of (confirmed1.begin (), confirmed1.end (), [&] (auto const & block) { + return ledger.dependents_confirmed (ledger.tx_begin_read (), *block); + })); + + // This should cement a few more dependencies + auto confirmed2 = ledger.confirm (ledger.tx_begin_write (), bottom->hash (), /* max cementing batch size */ 16); + ASSERT_FALSE (ledger.confirmed.block_exists (ledger.tx_begin_read (), bottom->hash ())); + ASSERT_EQ (confirmed2.size (), 16); + // Only topmost dependencies should get cemented during this call + ASSERT_TRUE (std::all_of (confirmed2.begin (), confirmed2.end (), [&] (auto const & block) { + return ledger.dependents_confirmed (ledger.tx_begin_read (), *block); + })); + + // A few more bounded calls should confirm the whole tree + auto confirmed3 = ledger.confirm (ledger.tx_begin_write (), bottom->hash (), /* max cementing batch size */ 64); + ASSERT_FALSE (ledger.confirmed.block_exists (ledger.tx_begin_read (), bottom->hash ())); + ASSERT_EQ (confirmed3.size (), 64); + // Only topmost dependencies should get cemented during this call + ASSERT_TRUE (std::all_of (confirmed2.begin (), confirmed2.end (), [&] (auto const & block) { + return ledger.dependents_confirmed (ledger.tx_begin_read (), *block); + })); + + auto confirmed4 = ledger.confirm (ledger.tx_begin_write (), bottom->hash (), /* max cementing batch size */ 64); + ASSERT_TRUE (ledger.confirmed.block_exists (ledger.tx_begin_read (), bottom->hash ())); + ASSERT_LT (confirmed4.size (), 64); + // Every block in the ledger should be cemented + ASSERT_TRUE (std::all_of (ctx.blocks ().begin (), ctx.blocks ().end (), [&] (auto const & block) { + return ledger.confirmed.block_exists (ledger.tx_begin_read (), block->hash ()); + })); +} + // Test that nullopt can be returned when there are no receivable entries TEST (ledger_receivable, upper_bound_account_none) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); ASSERT_EQ (ctx.ledger ().any.receivable_end (), ctx.ledger ().any.receivable_upper_bound (ctx.ledger ().tx_begin_read (), 0)); } // Test behavior of ledger::receivable_upper_bound when there are receivable entries for multiple accounts TEST (ledger_receivable, upper_bound_account_key) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); nano::block_builder builder; nano::keypair key; auto send1 = builder @@ -5581,7 +5676,7 @@ TEST (ledger_receivable, upper_bound_account_key) // Test that multiple receivable entries for the same account TEST (ledger_receivable, key_two) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); nano::block_builder builder; nano::keypair key; auto send1 = builder @@ -5620,13 +5715,13 @@ TEST (ledger_receivable, key_two) TEST (ledger_receivable, any_none) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); ASSERT_FALSE (ctx.ledger ().any.receivable_exists (ctx.ledger ().tx_begin_read (), nano::dev::genesis_key.pub)); } TEST (ledger_receivable, any_one) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); nano::block_builder builder; nano::keypair key; auto send1 = builder @@ -5646,7 +5741,7 @@ TEST (ledger_receivable, any_one) TEST (ledger_transaction, write_refresh) { - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); nano::block_builder builder; nano::keypair key; auto send1 = builder @@ -5683,7 +5778,7 @@ TEST (ledger_transaction, write_wait_order) { nano::test::system system; - auto ctx = nano::test::context::ledger_empty (); + auto ctx = nano::test::ledger_empty (); std::atomic acquired1{ false }; std::atomic acquired2{ false }; diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 1f6b9e74c1..25198d20d9 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -95,6 +95,7 @@ enum class detail { _invalid = 0, // Default value, should not be used + // common all, ok, test, @@ -124,6 +125,7 @@ enum class detail confirmed, unconfirmed, cemented, + cooldown, // processing queue queue, @@ -462,7 +464,9 @@ enum class detail // confirming_set notify_cemented, notify_already_cemented, + notify_intermediate, already_cemented, + cementing_hash, // election_state passive, diff --git a/nano/lib/thread_roles.cpp b/nano/lib/thread_roles.cpp index 2948caa217..2831c72038 100644 --- a/nano/lib/thread_roles.cpp +++ b/nano/lib/thread_roles.cpp @@ -61,7 +61,7 @@ std::string nano::thread_role::get_string (nano::thread_role::name role) case nano::thread_role::name::rpc_process_container: thread_role_name_string = "RPC process"; break; - case nano::thread_role::name::confirmation_height_processing: + case nano::thread_role::name::confirmation_height: thread_role_name_string = "Conf height"; break; case nano::thread_role::name::confirmation_height_notifications: diff --git a/nano/lib/thread_roles.hpp b/nano/lib/thread_roles.hpp index b6e3196cac..eef15632fa 100644 --- a/nano/lib/thread_roles.hpp +++ b/nano/lib/thread_roles.hpp @@ -25,7 +25,7 @@ enum class name signature_checking, rpc_request_processor, rpc_process_container, - confirmation_height_processing, + confirmation_height, confirmation_height_notifications, worker, bootstrap_worker, diff --git a/nano/node/confirming_set.cpp b/nano/node/confirming_set.cpp index ee72858b52..9a8f436e77 100644 --- a/nano/node/confirming_set.cpp +++ b/nano/node/confirming_set.cpp @@ -5,7 +5,8 @@ #include #include -nano::confirming_set::confirming_set (nano::ledger & ledger_a, nano::stats & stats_a) : +nano::confirming_set::confirming_set (confirming_set_config const & config_a, nano::ledger & ledger_a, nano::stats & stats_a) : + config{ config_a }, ledger{ ledger_a }, stats{ stats_a }, notification_workers{ 1, nano::thread_role::name::confirmation_height_notifications } @@ -13,14 +14,8 @@ nano::confirming_set::confirming_set (nano::ledger & ledger_a, nano::stats & sta batch_cemented.add ([this] (auto const & notification) { for (auto const & [block, confirmation_root] : notification.cemented) { - stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify_cemented); cemented_observers.notify (block); } - for (auto const & hash : notification.already_cemented) - { - stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify_already_cemented); - block_already_cemented_observers.notify (hash); - } }); } @@ -53,7 +48,7 @@ void nano::confirming_set::start () debug_assert (!thread.joinable ()); thread = std::thread{ [this] () { - nano::thread_role::set (nano::thread_role::name::confirmation_height_processing); + nano::thread_role::set (nano::thread_role::name::confirmation_height); run (); } }; } @@ -132,41 +127,72 @@ void nano::confirming_set::run_batch (std::unique_lock & lock) lock.unlock (); + auto notify = [this, &cemented, &already] () { + cemented_notification notification{}; + notification.cemented.swap (cemented); + notification.already_cemented.swap (already); + + // Wait for the worker thread if too many notifications are queued + while (notification_workers.num_queued_tasks () >= config.max_queued_notifications) + { + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::cooldown); + std::this_thread::sleep_for (100ms); + } + + notification_workers.push_task ([this, notification = std::move (notification)] () { + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify); + batch_cemented.notify (notification); + }); + }; + + // We might need to issue multiple notifications if the block we're confirming implicitly confirms more + auto notify_maybe = [this, &cemented, &already, ¬ify] (auto & transaction) { + if (cemented.size () >= config.max_blocks) + { + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify_intermediate); + transaction.commit (); + notify (); + transaction.renew (); + } + }; + { auto transaction = ledger.tx_begin_write ({ nano::tables::confirmation_height }, nano::store::writer::confirmation_height); - for (auto const & hash : batch) { - transaction.refresh_if_needed (); - - auto added = ledger.confirm (transaction, hash); - if (!added.empty ()) + do { - // Confirming this block may implicitly confirm more - for (auto & block : added) + transaction.refresh_if_needed (); + + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::cementing_hash); + + // Issue notifications here, so that `cemented` set is not too large before we add more blocks + notify_maybe (transaction); + + auto added = ledger.confirm (transaction, hash, config.max_blocks); + if (!added.empty ()) { - cemented.emplace_back (block, hash); + // Confirming this block may implicitly confirm more + stats.add (nano::stat::type::confirming_set, nano::stat::detail::cemented, added.size ()); + for (auto & block : added) + { + cemented.emplace_back (block, hash); + } } - - stats.add (nano::stat::type::confirming_set, nano::stat::detail::cemented, added.size ()); - } - else - { - already.push_back (hash); - stats.inc (nano::stat::type::confirming_set, nano::stat::detail::already_cemented); - } + else + { + stats.inc (nano::stat::type::confirming_set, nano::stat::detail::already_cemented); + already.push_back (hash); + debug_assert (ledger.confirmed.block_exists (transaction, hash)); + } + } while (!ledger.confirmed.block_exists (transaction, hash)); } } - cemented_notification notification{ - .cemented = std::move (cemented), - .already_cemented = std::move (already) - }; + notify (); - notification_workers.push_task ([this, notification = std::move (notification)] () { - stats.inc (nano::stat::type::confirming_set, nano::stat::detail::notify); - batch_cemented.notify (notification); - }); + release_assert (cemented.empty ()); + release_assert (already.empty ()); } std::unique_ptr nano::confirming_set::collect_container_info (std::string const & name) const diff --git a/nano/node/confirming_set.hpp b/nano/node/confirming_set.hpp index d6b8350de4..cb33915518 100644 --- a/nano/node/confirming_set.hpp +++ b/nano/node/confirming_set.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -12,14 +13,17 @@ namespace nano { -class node; -class block; -class ledger; -class stats; -} - -namespace nano +class confirming_set_config final { +public: + // TODO: Serialization & deserialization + +public: + /** Maximum number of dependent blocks to be stored in memory during processing */ + size_t max_blocks{ 64 * 128 }; + size_t max_queued_notifications{ 8 }; +}; + /** * Set of blocks to be durably confirmed */ @@ -29,13 +33,14 @@ class confirming_set final friend class confirmation_height_pruned_source_Test; public: - confirming_set (nano::ledger &, nano::stats &); + confirming_set (confirming_set_config const &, nano::ledger &, nano::stats &); ~confirming_set (); - // Adds a block to the set of blocks to be confirmed - void add (nano::block_hash const & hash); void start (); void stop (); + + // Adds a block to the set of blocks to be confirmed + void add (nano::block_hash const & hash); // Added blocks will remain in this set until after ledger has them marked as confirmed. bool exists (nano::block_hash const & hash) const; std::size_t size () const; @@ -54,16 +59,18 @@ class confirming_set final nano::observer_set batch_cemented; nano::observer_set> cemented_observers; - nano::observer_set block_already_cemented_observers; + +private: // Dependencies + confirming_set_config const & config; + nano::ledger & ledger; + nano::stats & stats; private: void run (); void run_batch (std::unique_lock &); std::deque next_batch (size_t max_count); - nano::ledger & ledger; - nano::stats & stats; - +private: std::unordered_set set; nano::thread_pool notification_workers; diff --git a/nano/node/fwd.hpp b/nano/node/fwd.hpp index 4a0f903a8a..293c803637 100644 --- a/nano/node/fwd.hpp +++ b/nano/node/fwd.hpp @@ -7,6 +7,7 @@ namespace nano { class active_elections; +class block; class confirming_set; class ledger; class local_block_broadcaster; diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 2bc1bbf513..05ea923576 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -187,7 +187,7 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy application_path (application_path_a), port_mapping (*this), block_processor (*this), - confirming_set_impl{ std::make_unique (ledger, stats) }, + confirming_set_impl{ std::make_unique (config.confirming_set, ledger, stats) }, confirming_set{ *confirming_set_impl }, active_impl{ std::make_unique (*this, confirming_set, block_processor) }, active{ *active_impl }, diff --git a/nano/node/nodeconfig.hpp b/nano/node/nodeconfig.hpp index f04b12ac5c..56be823c71 100644 --- a/nano/node/nodeconfig.hpp +++ b/nano/node/nodeconfig.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -153,6 +154,7 @@ class node_config nano::request_aggregator_config request_aggregator; nano::message_processor_config message_processor; nano::network_config network; + nano::confirming_set_config confirming_set; nano::local_block_broadcaster_config local_block_broadcaster; public: diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index df032b87eb..01d908cb44 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -804,38 +804,59 @@ nano::uint128_t nano::ledger::account_receivable (secure::transaction const & tr return result; } -std::deque> nano::ledger::confirm (secure::write_transaction const & transaction, nano::block_hash const & hash) +// Both stack and result set are bounded to limit maximum memory usage +// Callers must ensure that the target block was confirmed, and if not, call this function multiple times +std::deque> nano::ledger::confirm (secure::write_transaction const & transaction, nano::block_hash const & hash, size_t max_blocks) { std::deque> result; - std::stack stack; - stack.push (hash); + + std::deque stack; + stack.push_back (hash); while (!stack.empty ()) { - auto hash = stack.top (); + auto hash = stack.back (); auto block = any.block_get (transaction, hash); release_assert (block); + auto dependents = dependent_blocks (transaction, *block); for (auto const & dependent : dependents) { if (!dependent.is_zero () && !confirmed.block_exists_or_pruned (transaction, dependent)) { - stack.push (dependent); + stack.push_back (dependent); + + // Limit the stack size to avoid excessive memory usage + // This will forget the bottom of the dependency tree + if (stack.size () > max_blocks) + { + stack.pop_front (); + } } } - if (stack.top () == hash) + + if (stack.back () == hash) { - stack.pop (); + stack.pop_back (); if (!confirmed.block_exists_or_pruned (transaction, hash)) { - result.push_back (block); + // We must only confirm blocks that have their dependencies confirmed + debug_assert (dependents_confirmed (transaction, *block)); confirm (transaction, *block); + result.push_back (block); } } else { - // unconfirmed dependencies were added + // Unconfirmed dependencies were added + } + + // Early return might leave parts of the dependency tree unconfirmed + if (result.size () >= max_blocks) + { + break; } } + return result; } diff --git a/nano/secure/ledger.hpp b/nano/secure/ledger.hpp index 3e13256d29..99336c56ba 100644 --- a/nano/secure/ledger.hpp +++ b/nano/secure/ledger.hpp @@ -59,9 +59,9 @@ class ledger final std::string block_text (char const *); std::string block_text (nano::block_hash const &); std::pair hash_root_random (secure::transaction const &) const; - std::optional pending_info (secure::transaction const & transaction, nano::pending_key const & key) const; - std::deque> confirm (secure::write_transaction const & transaction, nano::block_hash const & hash); - nano::block_status process (secure::write_transaction const & transaction, std::shared_ptr block); + std::optional pending_info (secure::transaction const &, nano::pending_key const & key) const; + std::deque> confirm (secure::write_transaction const &, nano::block_hash const & hash, size_t max_blocks = 1024 * 128); + nano::block_status process (secure::write_transaction const &, std::shared_ptr block); bool rollback (secure::write_transaction const &, nano::block_hash const &, std::vector> &); bool rollback (secure::write_transaction const &, nano::block_hash const &); void update_account (secure::write_transaction const &, nano::account const &, nano::account_info const &, nano::account_info const &); @@ -70,26 +70,32 @@ class ledger final bool dependents_confirmed (secure::transaction const &, nano::block const &) const; bool is_epoch_link (nano::link const &) const; std::array dependent_blocks (secure::transaction const &, nano::block const &) const; - std::shared_ptr find_receive_block_by_send_hash (secure::transaction const & transaction, nano::account const & destination, nano::block_hash const & send_block_hash); + std::shared_ptr find_receive_block_by_send_hash (secure::transaction const &, nano::account const & destination, nano::block_hash const & send_block_hash); nano::account const & epoch_signer (nano::link const &) const; nano::link const & epoch_link (nano::epoch) const; bool migrate_lmdb_to_rocksdb (std::filesystem::path const &) const; bool bootstrap_weight_reached () const; static nano::epoch version (nano::block const & block); - nano::epoch version (secure::transaction const & transaction, nano::block_hash const & hash) const; - std::unique_ptr collect_container_info (std::string const & name) const; + nano::epoch version (secure::transaction const &, nano::block_hash const & hash) const; uint64_t cemented_count () const; uint64_t block_count () const; uint64_t account_count () const; uint64_t pruned_count () const; + + std::unique_ptr collect_container_info (std::string const & name) const; + +public: static nano::uint128_t const unit; + nano::ledger_constants & constants; nano::store::component & store; nano::ledger_cache cache; nano::stats & stats; + std::unordered_map bootstrap_weights; uint64_t bootstrap_weight_max_blocks{ 1 }; mutable std::atomic check_bootstrap_weights; + bool pruning{ false }; private: diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index 960a205a64..1ea6065a40 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -1145,7 +1145,8 @@ TEST (confirmation_height, many_accounts_send_receive_self_no_elections) nano::block_hash block_hash_being_processed{ 0 }; nano::store::write_queue write_queue{ false }; - nano::confirming_set confirming_set{ ledger, stats }; + nano::confirming_set_config confirming_set_config{}; + nano::confirming_set confirming_set{ confirming_set_config, ledger, stats }; auto const num_accounts = 100000; diff --git a/nano/test_common/CMakeLists.txt b/nano/test_common/CMakeLists.txt index 0f0b9d3c4a..56d6a53a4c 100644 --- a/nano/test_common/CMakeLists.txt +++ b/nano/test_common/CMakeLists.txt @@ -2,8 +2,8 @@ add_library( test_common chains.hpp chains.cpp - ledger.hpp - ledger.cpp + ledger_context.hpp + ledger_context.cpp make_store.hpp make_store.cpp network.hpp diff --git a/nano/test_common/ledger.cpp b/nano/test_common/ledger.cpp deleted file mode 100644 index 63ca93a2b4..0000000000 --- a/nano/test_common/ledger.cpp +++ /dev/null @@ -1,106 +0,0 @@ -#include -#include -#include -#include - -nano::test::context::ledger_context::ledger_context (std::deque> && blocks) : - store_m{ nano::make_store (logger, nano::unique_path (), nano::dev::constants) }, - stats_m{ logger }, - ledger_m{ *store_m, stats_m, nano::dev::constants }, - blocks_m{ blocks }, - pool_m{ nano::dev::network_params.network, 1 } -{ - debug_assert (!store_m->init_error ()); - auto tx = ledger_m.tx_begin_write (); - store_m->initialize (tx, ledger_m.cache, ledger_m.constants); - for (auto const & i : blocks_m) - { - auto process_result = ledger_m.process (tx, i); - debug_assert (process_result == nano::block_status::progress); - } -} - -nano::ledger & nano::test::context::ledger_context::ledger () -{ - return ledger_m; -} - -nano::store::component & nano::test::context::ledger_context::store () -{ - return *store_m; -} - -nano::stats & nano::test::context::ledger_context::stats () -{ - return stats_m; -} - -std::deque> const & nano::test::context::ledger_context::blocks () const -{ - return blocks_m; -} - -nano::work_pool & nano::test::context::ledger_context::pool () -{ - return pool_m; -} - -auto nano::test::context::ledger_empty () -> ledger_context -{ - return ledger_context{}; -} - -auto nano::test::context::ledger_send_receive () -> ledger_context -{ - std::deque> blocks; - nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; - nano::block_builder builder; - auto send = builder.state () - .make_block () - .account (nano::dev::genesis_key.pub) - .previous (nano::dev::genesis->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - 1) - .link (nano::dev::genesis_key.pub) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (nano::dev::genesis->hash ())) - .build (); - blocks.push_back (send); - auto receive = builder.state () - .make_block () - .account (nano::dev::genesis_key.pub) - .previous (send->hash ()) - .representative (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount) - .link (send->hash ()) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (send->hash ())) - .build (); - blocks.push_back (receive); - return ledger_context{ std::move (blocks) }; -} - -auto nano::test::context::ledger_send_receive_legacy () -> ledger_context -{ - std::deque> blocks; - nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; - nano::block_builder builder; - auto send = builder.send () - .make_block () - .previous (nano::dev::genesis->hash ()) - .destination (nano::dev::genesis_key.pub) - .balance (nano::dev::constants.genesis_amount - 1) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (nano::dev::genesis->hash ())) - .build (); - blocks.push_back (send); - auto receive = builder.receive () - .make_block () - .previous (send->hash ()) - .source (send->hash ()) - .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) - .work (*pool.generate (send->hash ())) - .build (); - blocks.push_back (receive); - return ledger_context{ std::move (blocks) }; -} diff --git a/nano/test_common/ledger.hpp b/nano/test_common/ledger.hpp deleted file mode 100644 index f996a9a4b9..0000000000 --- a/nano/test_common/ledger.hpp +++ /dev/null @@ -1,47 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace nano -{ -namespace store -{ - class component; -} -namespace test -{ - namespace context - { - class ledger_context - { - public: - /** 'blocks' initialises the ledger with each block in-order - Blocks must all return process_result::progress when processed */ - ledger_context (std::deque> && blocks = std::deque>{}); - nano::ledger & ledger (); - nano::store::component & store (); - nano::stats & stats (); - std::deque> const & blocks () const; - nano::work_pool & pool (); - - private: - nano::logger logger; - std::unique_ptr store_m; - nano::stats stats_m; - nano::ledger ledger_m; - std::deque> blocks_m; - nano::work_pool pool_m; - }; - - /** Only a genesis block */ - ledger_context ledger_empty (); - /** Send/receive pair of state blocks on the genesis account*/ - ledger_context ledger_send_receive (); - /** Send/receive pair of legacy blocks on the genesis account*/ - ledger_context ledger_send_receive_legacy (); - } -} -} diff --git a/nano/test_common/ledger_context.cpp b/nano/test_common/ledger_context.cpp new file mode 100644 index 0000000000..578aaa6342 --- /dev/null +++ b/nano/test_common/ledger_context.cpp @@ -0,0 +1,328 @@ +#include +#include +#include +#include + +nano::test::ledger_context::ledger_context (std::deque> && blocks) : + store_m{ nano::make_store (logger, nano::unique_path (), nano::dev::constants) }, + stats_m{ logger }, + ledger_m{ *store_m, stats_m, nano::dev::constants }, + blocks_m{ blocks }, + pool_m{ nano::dev::network_params.network, 1 } +{ + debug_assert (!store_m->init_error ()); + auto tx = ledger_m.tx_begin_write (); + store_m->initialize (tx, ledger_m.cache, ledger_m.constants); + for (auto const & i : blocks_m) + { + auto process_result = ledger_m.process (tx, i); + debug_assert (process_result == nano::block_status::progress, to_string (process_result)); + } +} + +nano::ledger & nano::test::ledger_context::ledger () +{ + return ledger_m; +} + +nano::store::component & nano::test::ledger_context::store () +{ + return *store_m; +} + +nano::stats & nano::test::ledger_context::stats () +{ + return stats_m; +} + +std::deque> const & nano::test::ledger_context::blocks () const +{ + return blocks_m; +} + +nano::work_pool & nano::test::ledger_context::pool () +{ + return pool_m; +} + +/* + * Ledger facotries + */ + +auto nano::test::ledger_empty () -> ledger_context +{ + return ledger_context{}; +} + +auto nano::test::ledger_send_receive () -> ledger_context +{ + std::deque> blocks; + nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; + nano::block_builder builder; + auto send = builder.state () + .make_block () + .account (nano::dev::genesis_key.pub) + .previous (nano::dev::genesis->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - 1) + .link (nano::dev::genesis_key.pub) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*pool.generate (nano::dev::genesis->hash ())) + .build (); + blocks.push_back (send); + auto receive = builder.state () + .make_block () + .account (nano::dev::genesis_key.pub) + .previous (send->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount) + .link (send->hash ()) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*pool.generate (send->hash ())) + .build (); + blocks.push_back (receive); + return ledger_context{ std::move (blocks) }; +} + +auto nano::test::ledger_send_receive_legacy () -> ledger_context +{ + std::deque> blocks; + nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; + nano::block_builder builder; + auto send = builder.send () + .make_block () + .previous (nano::dev::genesis->hash ()) + .destination (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - 1) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*pool.generate (nano::dev::genesis->hash ())) + .build (); + blocks.push_back (send); + auto receive = builder.receive () + .make_block () + .previous (send->hash ()) + .source (send->hash ()) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*pool.generate (send->hash ())) + .build (); + blocks.push_back (receive); + return ledger_context{ std::move (blocks) }; +} + +auto nano::test::ledger_diamond (unsigned height) -> ledger_context +{ + std::deque> blocks; + nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; + + using account_block_pair = std::pair>; + std::deque previous; + previous.push_back ({ nano::dev::genesis_key, nano::dev::genesis }); + + // Expanding tree + for (unsigned level = 0; level < height; ++level) + { + std::deque current; + while (!previous.empty ()) + { + auto const [key, root] = previous.front (); + previous.pop_front (); + + auto balance = root->balance_field ().value_or (nano::dev::constants.genesis_amount); + + nano::keypair target1, target2; + nano::block_builder builder; + + auto send1 = builder.state () + .make_block () + .account (key.pub) + .previous (root->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (balance.number () / 2) + .link (target1.pub) + .sign (key.prv, key.pub) + .work (*pool.generate (root->hash ())) + .build (); + + auto send2 = builder.state () + .make_block () + .account (key.pub) + .previous (send1->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (0) + .link (target2.pub) + .sign (key.prv, key.pub) + .work (*pool.generate (send1->hash ())) + .build (); + + auto open1 = builder.state () + .make_block () + .account (target1.pub) + .previous (0) + .representative (nano::dev::genesis_key.pub) + .balance (balance.number () - balance.number () / 2) + .link (send1->hash ()) + .sign (target1.prv, target1.pub) + .work (*pool.generate (target1.pub)) + .build (); + + auto open2 = builder.state () + .make_block () + .account (target2.pub) + .previous (0) + .representative (nano::dev::genesis_key.pub) + .balance (balance.number () / 2) + .link (send2->hash ()) + .sign (target2.prv, target2.pub) + .work (*pool.generate (target2.pub)) + .build (); + + blocks.push_back (send1); + blocks.push_back (send2); + blocks.push_back (open1); + blocks.push_back (open2); + + current.push_back ({ target1, open1 }); + current.push_back ({ target2, open2 }); + } + previous.clear (); + previous.swap (current); + } + + // Contracting tree + while (previous.size () > 1) + { + std::deque current; + while (!previous.empty ()) + { + auto const [key1, root1] = previous.front (); + previous.pop_front (); + auto const [key2, root2] = previous.front (); + previous.pop_front (); + + nano::keypair target; + nano::block_builder builder; + + auto balance1 = root1->balance_field ().value ().number (); + auto balance2 = root2->balance_field ().value ().number (); + + auto send1 = builder.state () + .make_block () + .account (key1.pub) + .previous (root1->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (0) + .link (target.pub) + .sign (key1.prv, key1.pub) + .work (*pool.generate (root1->hash ())) + .build (); + + auto send2 = builder.state () + .make_block () + .account (key2.pub) + .previous (root2->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (0) + .link (target.pub) + .sign (key2.prv, key2.pub) + .work (*pool.generate (root2->hash ())) + .build (); + + auto receive1 = builder.state () + .make_block () + .account (target.pub) + .previous (0) + .representative (nano::dev::genesis_key.pub) + .balance (balance1) + .link (send1->hash ()) + .sign (target.prv, target.pub) + .work (*pool.generate (target.pub)) + .build (); + + auto receive2 = builder.state () + .make_block () + .account (target.pub) + .previous (receive1->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (balance1 + balance2) + .link (send2->hash ()) + .sign (target.prv, target.pub) + .work (*pool.generate (receive1->hash ())) + .build (); + + blocks.push_back (send1); + blocks.push_back (send2); + blocks.push_back (receive1); + blocks.push_back (receive2); + + current.push_back ({ target, receive2 }); + } + previous.clear (); + previous.swap (current); + } + + return ledger_context{ std::move (blocks) }; +} + +auto nano::test::ledger_single_chain (unsigned height) -> nano::test::ledger_context +{ + std::deque> blocks; + nano::work_pool pool{ nano::dev::network_params.network, std::numeric_limits::max () }; + + nano::block_builder builder; + auto previous = nano::dev::genesis; + for (unsigned i = 0; i < height / 4; ++i) + { + auto send1 = builder.state () + .make_block () + .account (nano::dev::genesis_key.pub) + .previous (previous->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - 1) + .link (nano::dev::genesis_key.pub) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*pool.generate (previous->hash ())) + .build (); + + auto send2 = builder.state () + .make_block () + .account (nano::dev::genesis_key.pub) + .previous (send1->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - 2) + .link (nano::dev::genesis_key.pub) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*pool.generate (send1->hash ())) + .build (); + + auto receive1 = builder.state () + .make_block () + .account (nano::dev::genesis_key.pub) + .previous (send2->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount - 1) + .link (send1->hash ()) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*pool.generate (send2->hash ())) + .build (); + + auto receive2 = builder.state () + .make_block () + .account (nano::dev::genesis_key.pub) + .previous (receive1->hash ()) + .representative (nano::dev::genesis_key.pub) + .balance (nano::dev::constants.genesis_amount) + .link (send2->hash ()) + .sign (nano::dev::genesis_key.prv, nano::dev::genesis_key.pub) + .work (*pool.generate (receive1->hash ())) + .build (); + + blocks.push_back (send1); + blocks.push_back (send2); + blocks.push_back (receive1); + blocks.push_back (receive2); + + previous = receive2; + } + + return ledger_context{ std::move (blocks) }; +} \ No newline at end of file diff --git a/nano/test_common/ledger_context.hpp b/nano/test_common/ledger_context.hpp new file mode 100644 index 0000000000..8edca08508 --- /dev/null +++ b/nano/test_common/ledger_context.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace nano::test +{ +class ledger_context +{ +public: + /** 'blocks' initialises the ledger with each block in-order + Blocks must all return process_result::progress when processed */ + ledger_context (std::deque> && blocks = std::deque>{}); + nano::ledger & ledger (); + nano::store::component & store (); + nano::stats & stats (); + std::deque> const & blocks () const; + nano::work_pool & pool (); + +private: + nano::logger logger; + std::unique_ptr store_m; + nano::stats stats_m; + nano::ledger ledger_m; + std::deque> blocks_m; + nano::work_pool pool_m; +}; + +/** Only a genesis block */ +ledger_context ledger_empty (); +/** Send/receive pair of state blocks on the genesis account */ +ledger_context ledger_send_receive (); +/** Send/receive pair of legacy blocks on the genesis account */ +ledger_context ledger_send_receive_legacy (); +/** Full binary tree of state blocks */ +ledger_context ledger_diamond (unsigned height); +/** Single chain of state blocks with send and receives to itself */ +ledger_context ledger_single_chain (unsigned height); +}