From d61ea9f4c7d8966665eeab9bce78963c76eb3e8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Thu, 22 Aug 2024 17:10:37 +0200
Subject: [PATCH 01/14] Make functions non-const

---
 nano/node/node.cpp             |  2 +-
 nano/node/scheduler/hinted.cpp |  2 +-
 nano/node/scheduler/hinted.hpp |  2 +-
 nano/secure/transaction.hpp    |  4 ++--
 nano/store/transaction.cpp     |  8 ++++----
 nano/store/transaction.hpp     | 10 +++++-----
 6 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/nano/node/node.cpp b/nano/node/node.cpp
index fba2567204..9910a8a39b 100644
--- a/nano/node/node.cpp
+++ b/nano/node/node.cpp
@@ -975,7 +975,7 @@ bool nano::node::collect_ledger_pruning_targets (std::deque<nano::block_hash> &
 {
 	uint64_t read_operations (0);
 	bool finish_transaction (false);
-	auto const transaction = ledger.tx_begin_read ();
+	auto transaction = ledger.tx_begin_read ();
 	for (auto i (store.confirmation_height.begin (transaction, last_account_a)), n (store.confirmation_height.end ()); i != n && !finish_transaction;)
 	{
 		++read_operations;
diff --git a/nano/node/scheduler/hinted.cpp b/nano/node/scheduler/hinted.cpp
index a01fe97c8a..a507d5e51b 100644
--- a/nano/node/scheduler/hinted.cpp
+++ b/nano/node/scheduler/hinted.cpp
@@ -68,7 +68,7 @@ bool nano::scheduler::hinted::predicate () const
 	return active.vacancy (nano::election_behavior::hinted) > 0;
 }
 
-void nano::scheduler::hinted::activate (secure::read_transaction const & transaction, nano::block_hash const & hash, bool check_dependents)
+void nano::scheduler::hinted::activate (secure::read_transaction & transaction, nano::block_hash const & hash, bool check_dependents)
 {
 	const int max_iterations = 64;
 
diff --git a/nano/node/scheduler/hinted.hpp b/nano/node/scheduler/hinted.hpp
index e26772bda2..12eea04698 100644
--- a/nano/node/scheduler/hinted.hpp
+++ b/nano/node/scheduler/hinted.hpp
@@ -71,7 +71,7 @@ class hinted final
 	bool predicate () const;
 	void run ();
 	void run_iterative ();
-	void activate (secure::read_transaction const &, nano::block_hash const & hash, bool check_dependents);
+	void activate (secure::read_transaction &, nano::block_hash const & hash, bool check_dependents);
 
 	nano::uint128_t tally_threshold () const;
 	nano::uint128_t final_tally_threshold () const;
diff --git a/nano/secure/transaction.hpp b/nano/secure/transaction.hpp
index 5dd5e36d83..eedb7653d3 100644
--- a/nano/secure/transaction.hpp
+++ b/nano/secure/transaction.hpp
@@ -108,12 +108,12 @@ class read_transaction : public transaction
 		return txn;
 	}
 
-	void refresh () const
+	void refresh ()
 	{
 		txn.refresh ();
 	}
 
-	void refresh_if_needed (std::chrono::milliseconds max_age = std::chrono::milliseconds{ 500 }) const
+	void refresh_if_needed (std::chrono::milliseconds max_age = std::chrono::milliseconds{ 500 })
 	{
 		txn.refresh_if_needed (max_age);
 	}
diff --git a/nano/store/transaction.cpp b/nano/store/transaction.cpp
index 4cbc57e460..cba692175b 100644
--- a/nano/store/transaction.cpp
+++ b/nano/store/transaction.cpp
@@ -49,24 +49,24 @@ nano::id_dispenser::id_t nano::store::read_transaction::store_id () const
 	return impl->store_id;
 }
 
-void nano::store::read_transaction::reset () const
+void nano::store::read_transaction::reset ()
 {
 	impl->reset ();
 }
 
-void nano::store::read_transaction::renew () const
+void nano::store::read_transaction::renew ()
 {
 	impl->renew ();
 	start = std::chrono::steady_clock::now ();
 }
 
-void nano::store::read_transaction::refresh () const
+void nano::store::read_transaction::refresh ()
 {
 	reset ();
 	renew ();
 }
 
-void nano::store::read_transaction::refresh_if_needed (std::chrono::milliseconds max_age) const
+void nano::store::read_transaction::refresh_if_needed (std::chrono::milliseconds max_age)
 {
 	auto now = std::chrono::steady_clock::now ();
 	if (now - start > max_age)
diff --git a/nano/store/transaction.hpp b/nano/store/transaction.hpp
index 042b42bb59..e0b4b9eb64 100644
--- a/nano/store/transaction.hpp
+++ b/nano/store/transaction.hpp
@@ -53,14 +53,14 @@ class read_transaction final : public transaction
 	void * get_handle () const override;
 	nano::id_dispenser::id_t store_id () const override;
 
-	void reset () const;
-	void renew () const;
-	void refresh () const;
-	void refresh_if_needed (std::chrono::milliseconds max_age = std::chrono::milliseconds{ 500 }) const;
+	void reset ();
+	void renew ();
+	void refresh ();
+	void refresh_if_needed (std::chrono::milliseconds max_age = std::chrono::milliseconds{ 500 });
 
 private:
 	std::unique_ptr<read_transaction_impl> impl;
-	mutable std::chrono::steady_clock::time_point start;
+	std::chrono::steady_clock::time_point start;
 };
 
 /**

From e1695d49f1fd01f7dd6636d7d63e871c86b581cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Thu, 22 Aug 2024 17:08:18 +0200
Subject: [PATCH 02/14] Keep track of transaction epoch

---
 nano/store/iterator_impl.hpp    | 20 ++++++++++++++++++--
 nano/store/lmdb/iterator.hpp    |  3 ++-
 nano/store/rocksdb/iterator.hpp |  3 ++-
 nano/store/transaction.cpp      | 13 +++++++++++++
 nano/store/transaction.hpp      |  7 +++++++
 5 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/nano/store/iterator_impl.hpp b/nano/store/iterator_impl.hpp
index 0f873e976b..48130d14ed 100644
--- a/nano/store/iterator_impl.hpp
+++ b/nano/store/iterator_impl.hpp
@@ -1,5 +1,8 @@
 #pragma once
 
+#include <nano/lib/utility.hpp>
+#include <nano/store/transaction.hpp>
+
 #include <utility>
 
 namespace nano::store
@@ -8,7 +11,16 @@ template <typename T, typename U>
 class iterator_impl
 {
 public:
-	virtual ~iterator_impl () = default;
+	explicit iterator_impl (nano::store::transaction const & transaction_a) :
+		transaction{ transaction_a },
+		transaction_epoch{ transaction_a.epoch () }
+	{
+	}
+	virtual ~iterator_impl ()
+	{
+		debug_assert (transaction_epoch == transaction.epoch (), "invalid iterator-transaction lifetime detected");
+	}
+
 	virtual iterator_impl<T, U> & operator++ () = 0;
 	virtual iterator_impl<T, U> & operator-- () = 0;
 	virtual bool operator== (iterator_impl<T, U> const & other_a) const = 0;
@@ -23,5 +35,9 @@ class iterator_impl
 	{
 		return !(*this == other_a);
 	}
+
+protected:
+	nano::store::transaction const & transaction;
+	nano::store::transaction::epoch_t const transaction_epoch;
 };
-} // namespace nano::store
+}
diff --git a/nano/store/lmdb/iterator.hpp b/nano/store/lmdb/iterator.hpp
index ff7cb0977c..3c2fe197c7 100644
--- a/nano/store/lmdb/iterator.hpp
+++ b/nano/store/lmdb/iterator.hpp
@@ -14,7 +14,8 @@ template <typename T, typename U>
 class iterator : public iterator_impl<T, U>
 {
 public:
-	iterator (store::transaction const & transaction_a, env const & env_a, MDB_dbi db_a, MDB_val const & val_a = MDB_val{}, bool const direction_asc = true)
+	iterator (store::transaction const & transaction_a, env const & env_a, MDB_dbi db_a, MDB_val const & val_a = MDB_val{}, bool const direction_asc = true) :
+		nano::store::iterator_impl<T, U> (transaction_a)
 	{
 		auto status (mdb_cursor_open (env_a.tx (transaction_a), db_a, &cursor));
 		release_assert (status == 0);
diff --git a/nano/store/rocksdb/iterator.hpp b/nano/store/rocksdb/iterator.hpp
index ec4865e526..b661c8af6e 100644
--- a/nano/store/rocksdb/iterator.hpp
+++ b/nano/store/rocksdb/iterator.hpp
@@ -33,7 +33,8 @@ class iterator : public iterator_impl<T, U>
 public:
 	iterator () = default;
 
-	iterator (::rocksdb::DB * db, store::transaction const & transaction_a, ::rocksdb::ColumnFamilyHandle * handle_a, db_val const * val_a, bool const direction_asc)
+	iterator (::rocksdb::DB * db, store::transaction const & transaction_a, ::rocksdb::ColumnFamilyHandle * handle_a, db_val const * val_a, bool const direction_asc) :
+		nano::store::iterator_impl<T, U> (transaction_a)
 	{
 		// Don't fill the block cache for any blocks read as a result of an iterator
 		if (is_read (transaction_a))
diff --git a/nano/store/transaction.cpp b/nano/store/transaction.cpp
index cba692175b..2609c231b6 100644
--- a/nano/store/transaction.cpp
+++ b/nano/store/transaction.cpp
@@ -29,6 +29,15 @@ nano::store::write_transaction_impl::write_transaction_impl (nano::id_dispenser:
 {
 }
 
+/*
+ * transaction
+ */
+
+auto nano::store::transaction::epoch () const -> epoch_t
+{
+	return current_epoch;
+}
+
 /*
  * read_transaction
  */
@@ -51,11 +60,13 @@ nano::id_dispenser::id_t nano::store::read_transaction::store_id () const
 
 void nano::store::read_transaction::reset ()
 {
+	++current_epoch;
 	impl->reset ();
 }
 
 void nano::store::read_transaction::renew ()
 {
+	++current_epoch;
 	impl->renew ();
 	start = std::chrono::steady_clock::now ();
 }
@@ -102,11 +113,13 @@ nano::id_dispenser::id_t nano::store::write_transaction::store_id () const
 
 void nano::store::write_transaction::commit ()
 {
+	++current_epoch;
 	impl->commit ();
 }
 
 void nano::store::write_transaction::renew ()
 {
+	++current_epoch;
 	impl->renew ();
 	start = std::chrono::steady_clock::now ();
 }
diff --git a/nano/store/transaction.hpp b/nano/store/transaction.hpp
index e0b4b9eb64..79e96cdcae 100644
--- a/nano/store/transaction.hpp
+++ b/nano/store/transaction.hpp
@@ -36,10 +36,17 @@ class write_transaction_impl : public transaction_impl
 
 class transaction
 {
+public:
+	using epoch_t = size_t;
+
 public:
 	virtual ~transaction () = default;
 	virtual void * get_handle () const = 0;
 	virtual nano::id_dispenser::id_t store_id () const = 0;
+	epoch_t epoch () const;
+
+protected:
+	epoch_t current_epoch{ 0 };
 };
 
 /**

From 968d2d008c8917ed28d4983ce9dd6957be15f059 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Thu, 22 Aug 2024 23:16:52 +0200
Subject: [PATCH 03/14] Revert backlog population transaction refresh

---
 nano/node/backlog_population.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/nano/node/backlog_population.cpp b/nano/node/backlog_population.cpp
index d938273a38..c49a5ebc95 100644
--- a/nano/node/backlog_population.cpp
+++ b/nano/node/backlog_population.cpp
@@ -100,8 +100,6 @@ void nano::backlog_population::populate_backlog (nano::unique_lock<nano::mutex>
 			auto const end = ledger.store.account.end ();
 			for (; i != end && count < chunk_size; ++i, ++count, ++total)
 			{
-				transaction.refresh_if_needed ();
-
 				stats.inc (nano::stat::type::backlog, nano::stat::detail::total);
 
 				auto const & account = i->first;

From 16e927034e9f69a9fbb2212c7936a7b6b77672c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Fri, 23 Aug 2024 10:05:18 +0200
Subject: [PATCH 04/14] Transaction timestamp accessor

---
 nano/secure/transaction.hpp | 10 ++++++++++
 nano/store/transaction.cpp  |  5 +++++
 nano/store/transaction.hpp  |  5 +++--
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/nano/secure/transaction.hpp b/nano/secure/transaction.hpp
index eedb7653d3..6c4acde3d3 100644
--- a/nano/secure/transaction.hpp
+++ b/nano/secure/transaction.hpp
@@ -79,6 +79,11 @@ class write_transaction : public transaction
 		return false;
 	}
 
+	auto timestamp () const
+	{
+		return txn.timestamp ();
+	}
+
 	// Conversion operator to const nano::store::transaction&
 	operator const nano::store::transaction & () const override
 	{
@@ -118,6 +123,11 @@ class read_transaction : public transaction
 		txn.refresh_if_needed (max_age);
 	}
 
+	auto timestamp () const
+	{
+		return txn.timestamp ();
+	}
+
 	// Conversion operator to const nano::store::transaction&
 	operator const nano::store::transaction & () const override
 	{
diff --git a/nano/store/transaction.cpp b/nano/store/transaction.cpp
index 2609c231b6..dcfc3d7e8d 100644
--- a/nano/store/transaction.cpp
+++ b/nano/store/transaction.cpp
@@ -38,6 +38,11 @@ auto nano::store::transaction::epoch () const -> epoch_t
 	return current_epoch;
 }
 
+std::chrono::steady_clock::time_point nano::store::transaction::timestamp () const
+{
+	return start;
+}
+
 /*
  * read_transaction
  */
diff --git a/nano/store/transaction.hpp b/nano/store/transaction.hpp
index 79e96cdcae..23459a9258 100644
--- a/nano/store/transaction.hpp
+++ b/nano/store/transaction.hpp
@@ -43,10 +43,13 @@ class transaction
 	virtual ~transaction () = default;
 	virtual void * get_handle () const = 0;
 	virtual nano::id_dispenser::id_t store_id () const = 0;
+
 	epoch_t epoch () const;
+	std::chrono::steady_clock::time_point timestamp () const;
 
 protected:
 	epoch_t current_epoch{ 0 };
+	std::chrono::steady_clock::time_point start{};
 };
 
 /**
@@ -67,7 +70,6 @@ class read_transaction final : public transaction
 
 private:
 	std::unique_ptr<read_transaction_impl> impl;
-	std::chrono::steady_clock::time_point start;
 };
 
 /**
@@ -89,6 +91,5 @@ class write_transaction final : public transaction
 
 private:
 	std::unique_ptr<write_transaction_impl> impl;
-	std::chrono::steady_clock::time_point start;
 };
 } // namespace nano::store

From 92df37bc053ade591d24c41de1dd00b2abd8a162 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Fri, 23 Aug 2024 10:12:40 +0200
Subject: [PATCH 05/14] Avoid long held transactions during backlog population

---
 nano/node/backlog_population.cpp | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/nano/node/backlog_population.cpp b/nano/node/backlog_population.cpp
index c49a5ebc95..89965282fb 100644
--- a/nano/node/backlog_population.cpp
+++ b/nano/node/backlog_population.cpp
@@ -95,19 +95,26 @@ void nano::backlog_population::populate_backlog (nano::unique_lock<nano::mutex>
 		{
 			auto transaction = ledger.tx_begin_read ();
 
-			auto count = 0u;
-			auto i = ledger.store.account.begin (transaction, next);
+			auto it = ledger.store.account.begin (transaction, next);
 			auto const end = ledger.store.account.end ();
-			for (; i != end && count < chunk_size; ++i, ++count, ++total)
+
+			auto should_refresh = [&transaction] () {
+				auto cutoff = std::chrono::steady_clock::now () - 100ms; // TODO: Make this configurable
+				return transaction.timestamp () < cutoff;
+			};
+
+			for (size_t count = 0; it != end && count < chunk_size && !should_refresh (); ++it, ++count, ++total)
 			{
 				stats.inc (nano::stat::type::backlog, nano::stat::detail::total);
 
-				auto const & account = i->first;
-				auto const & account_info = i->second;
+				auto const & account = it->first;
+				auto const & account_info = it->second;
+
 				activate (transaction, account, account_info);
 
 				next = account.number () + 1;
 			}
+
 			done = ledger.store.account.begin (transaction, next) == end;
 		}
 

From 0ee826aba3f2c9535dae9fbc5a2f472184ee97e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Fri, 23 Aug 2024 11:23:40 +0200
Subject: [PATCH 06/14] Mark `collect_ledger_pruning_targets` as FIXME

---
 nano/node/node.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nano/node/node.cpp b/nano/node/node.cpp
index 9910a8a39b..00f6fcafa4 100644
--- a/nano/node/node.cpp
+++ b/nano/node/node.cpp
@@ -1003,6 +1003,7 @@ bool nano::node::collect_ledger_pruning_targets (std::deque<nano::block_hash> &
 			}
 			if (++depth % batch_read_size_a == 0)
 			{
+				// FIXME: This is triggering an assertion where the iterator is still used after transaction is refreshed
 				transaction.refresh ();
 			}
 		}

From 6420c0a1049e50722f8713abb448d4d3aab11c2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Fri, 23 Aug 2024 15:01:54 +0200
Subject: [PATCH 07/14] Fix `nano::node::ongoing_bootstrap ()` transaction
 lifetime

---
 nano/node/node.cpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/nano/node/node.cpp b/nano/node/node.cpp
index 00f6fcafa4..adeb925acb 100644
--- a/nano/node/node.cpp
+++ b/nano/node/node.cpp
@@ -891,10 +891,13 @@ void nano::node::ongoing_bootstrap ()
 		{
 			// Find last online weight sample (last active time for node)
 			uint64_t last_sample_time (0);
-			auto last_record = store.online_weight.rbegin (store.tx_begin_read ());
-			if (last_record != store.online_weight.end ())
 			{
-				last_sample_time = last_record->first;
+				auto transaction = store.tx_begin_read ();
+				auto last_record = store.online_weight.rbegin (transaction);
+				if (last_record != store.online_weight.end ())
+				{
+					last_sample_time = last_record->first;
+				}
 			}
 			uint64_t time_since_last_sample = std::chrono::duration_cast<std::chrono::seconds> (std::chrono::system_clock::now ().time_since_epoch ()).count () - static_cast<uint64_t> (last_sample_time / std::pow (10, 9)); // Nanoseconds to seconds
 			if (time_since_last_sample + 60 * 60 < std::numeric_limits<uint32_t>::max ())

From cb8c901a26f559cd1bcb0fa617d5fd1b9731f47b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Mon, 26 Aug 2024 18:59:52 +0200
Subject: [PATCH 08/14] Disable failing prunning tests

---
 nano/core_test/node.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp
index 34af749f6b..f48fe70c79 100644
--- a/nano/core_test/node.cpp
+++ b/nano/core_test/node.cpp
@@ -3592,7 +3592,7 @@ TEST (node, pruning_automatic)
 	ASSERT_TRUE (nano::test::block_or_pruned_all_exists (node1, { nano::dev::genesis, send1, send2 }));
 }
 
-TEST (node, pruning_age)
+TEST (node, DISABLED_pruning_age)
 {
 	nano::test::system system{};
 
@@ -3653,7 +3653,7 @@ TEST (node, pruning_age)
 
 // Test that a node configured with `enable_pruning` will
 // prune DEEP-enough confirmed blocks by explicitly saying `node.ledger_pruning` in the unit test
-TEST (node, pruning_depth)
+TEST (node, DISABLED_pruning_depth)
 {
 	nano::test::system system{};
 

From c15fedb7109d1c392f63bc733a62e128ef3be06b Mon Sep 17 00:00:00 2001
From: RickiNano <81099017+RickiNano@users.noreply.github.com>
Date: Thu, 22 Aug 2024 11:11:08 +0200
Subject: [PATCH 09/14] Drop existing rep_weights table before migrating

---
 nano/store/lmdb/lmdb.cpp       |  1 +
 nano/store/rocksdb/rocksdb.cpp | 16 ++++++++++------
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp
index 32cde114f2..d25aaa5235 100644
--- a/nano/store/lmdb/lmdb.cpp
+++ b/nano/store/lmdb/lmdb.cpp
@@ -255,6 +255,7 @@ void nano::store::lmdb::component::upgrade_v21_to_v22 (store::write_transaction
 void nano::store::lmdb::component::upgrade_v22_to_v23 (store::write_transaction const & transaction_a)
 {
 	logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23...");
+	drop (transaction_a, tables::rep_weights);
 	auto i{ make_iterator<nano::account, nano::account_info_v22> (transaction_a, tables::accounts) };
 	auto end{ store::iterator<nano::account, nano::account_info_v22> (nullptr) };
 	uint64_t processed_accounts = 0;
diff --git a/nano/store/rocksdb/rocksdb.cpp b/nano/store/rocksdb/rocksdb.cpp
index 6ae1f43d99..a403ac93a0 100644
--- a/nano/store/rocksdb/rocksdb.cpp
+++ b/nano/store/rocksdb/rocksdb.cpp
@@ -289,14 +289,18 @@ void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transacti
 {
 	logger.info (nano::log::type::rocksdb, "Upgrading database from v22 to v23...");
 
-	if (!column_family_exists ("rep_weights"))
+	if (column_family_exists ("rep_weights"))
 	{
-		logger.info (nano::log::type::rocksdb, "Creating table rep_weights");
-		::rocksdb::ColumnFamilyOptions new_cf_options;
-		::rocksdb::ColumnFamilyHandle * new_cf_handle;
-		::rocksdb::Status status = db->CreateColumnFamily (new_cf_options, "rep_weights", &new_cf_handle);
-		handles.emplace_back (new_cf_handle);
+		logger.info (nano::log::type::rocksdb, "Dropping existing rep_weights table");
+		drop (transaction_a, tables::rep_weights);
 	}
+
+	logger.info (nano::log::type::rocksdb, "Creating table rep_weights");
+	::rocksdb::ColumnFamilyOptions new_cf_options;
+	::rocksdb::ColumnFamilyHandle * new_cf_handle;
+	::rocksdb::Status status = db->CreateColumnFamily (new_cf_options, "rep_weights", &new_cf_handle);
+	handles.emplace_back (new_cf_handle);
+
 	auto i{ make_iterator<nano::account, nano::account_info_v22> (transaction_a, tables::accounts) };
 	auto end{ store::iterator<nano::account, nano::account_info_v22> (nullptr) };
 	uint64_t processed_accounts = 0;

From aa167a34cef236d5cf0ae890d5932869c4fe0e80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Thu, 22 Aug 2024 16:23:55 +0200
Subject: [PATCH 10/14] Remove unnecessary transaction constness

---
 nano/store/lmdb/lmdb.cpp       | 38 +++++++++++++++++-----------------
 nano/store/lmdb/lmdb.hpp       |  6 +++---
 nano/store/rocksdb/rocksdb.cpp | 30 +++++++++++++--------------
 nano/store/rocksdb/rocksdb.hpp |  8 +++----
 4 files changed, 41 insertions(+), 41 deletions(-)

diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp
index d25aaa5235..6c3c5a798b 100644
--- a/nano/store/lmdb/lmdb.cpp
+++ b/nano/store/lmdb/lmdb.cpp
@@ -209,10 +209,10 @@ void nano::store::lmdb::component::open_databases (bool & error_a, store::transa
 	error_a |= mdb_dbi_open (env.tx (transaction_a), "rep_weights", flags, &rep_weight_store.rep_weights_handle) != 0;
 }
 
-bool nano::store::lmdb::component::do_upgrades (store::write_transaction & transaction_a, nano::ledger_constants & constants, bool & needs_vacuuming)
+bool nano::store::lmdb::component::do_upgrades (store::write_transaction & transaction, nano::ledger_constants & constants, bool & needs_vacuuming)
 {
 	auto error (false);
-	auto version_l = version.get (transaction_a);
+	auto version_l = version.get (transaction);
 	if (version_l < version_minimum)
 	{
 		logger.critical (nano::log::type::lmdb, "The version of the ledger ({}) is lower than the minimum ({}) which is supported for upgrades. Either upgrade a node first or delete the ledger.", version_l, version_minimum);
@@ -221,13 +221,13 @@ bool nano::store::lmdb::component::do_upgrades (store::write_transaction & trans
 	switch (version_l)
 	{
 		case 21:
-			upgrade_v21_to_v22 (transaction_a);
+			upgrade_v21_to_v22 (transaction);
 			[[fallthrough]];
 		case 22:
-			upgrade_v22_to_v23 (transaction_a);
+			upgrade_v22_to_v23 (transaction);
 			[[fallthrough]];
 		case 23:
-			upgrade_v23_to_v24 (transaction_a);
+			upgrade_v23_to_v24 (transaction);
 			[[fallthrough]];
 		case 24:
 			break;
@@ -239,24 +239,24 @@ bool nano::store::lmdb::component::do_upgrades (store::write_transaction & trans
 	return error;
 }
 
-void nano::store::lmdb::component::upgrade_v21_to_v22 (store::write_transaction const & transaction_a)
+void nano::store::lmdb::component::upgrade_v21_to_v22 (store::write_transaction & transaction)
 {
 	logger.info (nano::log::type::lmdb, "Upgrading database from v21 to v22...");
 
 	MDB_dbi unchecked_handle{ 0 };
-	release_assert (!mdb_dbi_open (env.tx (transaction_a), "unchecked", MDB_CREATE, &unchecked_handle));
-	release_assert (!mdb_drop (env.tx (transaction_a), unchecked_handle, 1)); // del = 1, to delete it from the environment and close the DB handle.
-	version.put (transaction_a, 22);
+	release_assert (!mdb_dbi_open (env.tx (transaction), "unchecked", MDB_CREATE, &unchecked_handle));
+	release_assert (!mdb_drop (env.tx (transaction), unchecked_handle, 1)); // del = 1, to delete it from the environment and close the DB handle.
+	version.put (transaction, 22);
 
 	logger.info (nano::log::type::lmdb, "Upgrading database from v21 to v22 completed");
 }
 
 // Fill rep_weights table with all existing representatives and their vote weight
-void nano::store::lmdb::component::upgrade_v22_to_v23 (store::write_transaction const & transaction_a)
+void nano::store::lmdb::component::upgrade_v22_to_v23 (store::write_transaction & transaction)
 {
 	logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23...");
-	drop (transaction_a, tables::rep_weights);
-	auto i{ make_iterator<nano::account, nano::account_info_v22> (transaction_a, tables::accounts) };
+	drop (transaction, tables::rep_weights);
+	auto i{ make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts) };
 	auto end{ store::iterator<nano::account, nano::account_info_v22> (nullptr) };
 	uint64_t processed_accounts = 0;
 	for (; i != end; ++i)
@@ -265,13 +265,13 @@ void nano::store::lmdb::component::upgrade_v22_to_v23 (store::write_transaction
 		{
 			nano::uint128_t total{ 0 };
 			nano::store::lmdb::db_val value;
-			auto status = get (transaction_a, tables::rep_weights, i->second.representative, value);
+			auto status = get (transaction, tables::rep_weights, i->second.representative, value);
 			if (success (status))
 			{
 				total = nano::amount{ value }.number ();
 			}
 			total += i->second.balance.number ();
-			status = put (transaction_a, tables::rep_weights, i->second.representative, nano::amount{ total });
+			status = put (transaction, tables::rep_weights, i->second.representative, nano::amount{ total });
 			release_assert_success (status);
 		}
 		processed_accounts++;
@@ -281,18 +281,18 @@ void nano::store::lmdb::component::upgrade_v22_to_v23 (store::write_transaction
 		}
 	}
 	logger.info (nano::log::type::lmdb, "Processed {} accounts", processed_accounts);
-	version.put (transaction_a, 23);
+	version.put (transaction, 23);
 	logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23 completed");
 }
 
-void nano::store::lmdb::component::upgrade_v23_to_v24 (store::write_transaction const & transaction_a)
+void nano::store::lmdb::component::upgrade_v23_to_v24 (store::write_transaction & transaction)
 {
 	logger.info (nano::log::type::lmdb, "Upgrading database from v23 to v24...");
 
 	MDB_dbi frontiers_handle{ 0 };
-	release_assert (!mdb_dbi_open (env.tx (transaction_a), "frontiers", MDB_CREATE, &frontiers_handle));
-	release_assert (!mdb_drop (env.tx (transaction_a), frontiers_handle, 1)); // del = 1, to delete it from the environment and close the DB handle.
-	version.put (transaction_a, 24);
+	release_assert (!mdb_dbi_open (env.tx (transaction), "frontiers", MDB_CREATE, &frontiers_handle));
+	release_assert (!mdb_drop (env.tx (transaction), frontiers_handle, 1)); // del = 1, to delete it from the environment and close the DB handle.
+	version.put (transaction, 24);
 	logger.info (nano::log::type::lmdb, "Upgrading database from v23 to v24 completed");
 }
 
diff --git a/nano/store/lmdb/lmdb.hpp b/nano/store/lmdb/lmdb.hpp
index 8cf9d94bc3..290b19f813 100644
--- a/nano/store/lmdb/lmdb.hpp
+++ b/nano/store/lmdb/lmdb.hpp
@@ -112,9 +112,9 @@ class component : public nano::store::component
 
 private:
 	bool do_upgrades (store::write_transaction &, nano::ledger_constants & constants, bool &);
-	void upgrade_v21_to_v22 (store::write_transaction const &);
-	void upgrade_v22_to_v23 (store::write_transaction const &);
-	void upgrade_v23_to_v24 (store::write_transaction const &);
+	void upgrade_v21_to_v22 (store::write_transaction &);
+	void upgrade_v22_to_v23 (store::write_transaction &);
+	void upgrade_v23_to_v24 (store::write_transaction &);
 
 	void open_databases (bool &, store::transaction const &, unsigned);
 
diff --git a/nano/store/rocksdb/rocksdb.cpp b/nano/store/rocksdb/rocksdb.cpp
index a403ac93a0..2bd4aae5f2 100644
--- a/nano/store/rocksdb/rocksdb.cpp
+++ b/nano/store/rocksdb/rocksdb.cpp
@@ -210,10 +210,10 @@ void nano::store::rocksdb::component::open (bool & error_a, std::filesystem::pat
 	error_a |= !s.ok ();
 }
 
-bool nano::store::rocksdb::component::do_upgrades (store::write_transaction const & transaction_a)
+bool nano::store::rocksdb::component::do_upgrades (store::write_transaction & transaction)
 {
 	bool error_l{ false };
-	auto version_l = version.get (transaction_a);
+	auto version_l = version.get (transaction);
 	switch (version_l)
 	{
 		case 1:
@@ -240,13 +240,13 @@ bool nano::store::rocksdb::component::do_upgrades (store::write_transaction cons
 		case 19:
 		case 20:
 		case 21:
-			upgrade_v21_to_v22 (transaction_a);
+			upgrade_v21_to_v22 (transaction);
 			[[fallthrough]];
 		case 22:
-			upgrade_v22_to_v23 (transaction_a);
+			upgrade_v22_to_v23 (transaction);
 			[[fallthrough]];
 		case 23:
-			upgrade_v23_to_v24 (transaction_a);
+			upgrade_v23_to_v24 (transaction);
 			[[fallthrough]];
 		case 24:
 			break;
@@ -258,7 +258,7 @@ bool nano::store::rocksdb::component::do_upgrades (store::write_transaction cons
 	return error_l;
 }
 
-void nano::store::rocksdb::component::upgrade_v21_to_v22 (store::write_transaction const & transaction_a)
+void nano::store::rocksdb::component::upgrade_v21_to_v22 (store::write_transaction & transaction)
 {
 	logger.info (nano::log::type::rocksdb, "Upgrading database from v21 to v22...");
 
@@ -279,20 +279,20 @@ void nano::store::rocksdb::component::upgrade_v21_to_v22 (store::write_transacti
 		logger.debug (nano::log::type::rocksdb, "Finished removing unchecked table");
 	}
 
-	version.put (transaction_a, 22);
+	version.put (transaction, 22);
 
 	logger.info (nano::log::type::rocksdb, "Upgrading database from v21 to v22 completed");
 }
 
 // Fill rep_weights table with all existing representatives and their vote weight
-void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transaction const & transaction_a)
+void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transaction & transaction)
 {
 	logger.info (nano::log::type::rocksdb, "Upgrading database from v22 to v23...");
 
 	if (column_family_exists ("rep_weights"))
 	{
 		logger.info (nano::log::type::rocksdb, "Dropping existing rep_weights table");
-		drop (transaction_a, tables::rep_weights);
+		drop (transaction, tables::rep_weights);
 	}
 
 	logger.info (nano::log::type::rocksdb, "Creating table rep_weights");
@@ -301,7 +301,7 @@ void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transacti
 	::rocksdb::Status status = db->CreateColumnFamily (new_cf_options, "rep_weights", &new_cf_handle);
 	handles.emplace_back (new_cf_handle);
 
-	auto i{ make_iterator<nano::account, nano::account_info_v22> (transaction_a, tables::accounts) };
+	auto i{ make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts) };
 	auto end{ store::iterator<nano::account, nano::account_info_v22> (nullptr) };
 	uint64_t processed_accounts = 0;
 	for (; i != end; ++i)
@@ -310,13 +310,13 @@ void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transacti
 		{
 			nano::uint128_t total{ 0 };
 			nano::store::rocksdb::db_val value;
-			auto status = get (transaction_a, tables::rep_weights, i->second.representative, value);
+			auto status = get (transaction, tables::rep_weights, i->second.representative, value);
 			if (success (status))
 			{
 				total = nano::amount{ value }.number ();
 			}
 			total += i->second.balance.number ();
-			status = put (transaction_a, tables::rep_weights, i->second.representative, nano::amount{ total });
+			status = put (transaction, tables::rep_weights, i->second.representative, nano::amount{ total });
 			release_assert_success (status);
 		}
 		processed_accounts++;
@@ -326,11 +326,11 @@ void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transacti
 		}
 	}
 	logger.info (nano::log::type::rocksdb, "Processed {} accounts", processed_accounts);
-	version.put (transaction_a, 23);
+	version.put (transaction, 23);
 	logger.info (nano::log::type::rocksdb, "Upgrading database from v22 to v23 completed");
 }
 
-void nano::store::rocksdb::component::upgrade_v23_to_v24 (store::write_transaction const & transaction_a)
+void nano::store::rocksdb::component::upgrade_v23_to_v24 (store::write_transaction & transaction)
 {
 	logger.info (nano::log::type::rocksdb, "Upgrading database from v23 to v24...");
 
@@ -351,7 +351,7 @@ void nano::store::rocksdb::component::upgrade_v23_to_v24 (store::write_transacti
 		logger.debug (nano::log::type::rocksdb, "Finished removing frontiers table");
 	}
 
-	version.put (transaction_a, 24);
+	version.put (transaction, 24);
 	logger.info (nano::log::type::rocksdb, "Upgrading database from v23 to v24 completed");
 }
 
diff --git a/nano/store/rocksdb/rocksdb.hpp b/nano/store/rocksdb/rocksdb.hpp
index 0b6f11cb85..5d8b22bba7 100644
--- a/nano/store/rocksdb/rocksdb.hpp
+++ b/nano/store/rocksdb/rocksdb.hpp
@@ -148,10 +148,10 @@ class component : public nano::store::component
 
 	void open (bool & error_a, std::filesystem::path const & path_a, bool open_read_only_a, ::rocksdb::Options const & options_a, std::vector<::rocksdb::ColumnFamilyDescriptor> column_families);
 
-	bool do_upgrades (store::write_transaction const &);
-	void upgrade_v21_to_v22 (store::write_transaction const &);
-	void upgrade_v22_to_v23 (store::write_transaction const &);
-	void upgrade_v23_to_v24 (store::write_transaction const &);
+	bool do_upgrades (store::write_transaction &);
+	void upgrade_v21_to_v22 (store::write_transaction &);
+	void upgrade_v22_to_v23 (store::write_transaction &);
+	void upgrade_v23_to_v24 (store::write_transaction &);
 
 	void construct_column_family_mutexes ();
 	::rocksdb::Options get_db_options ();

From b2108b71aae62f8dcea0fc7c7a1d47a2cc85459d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Mon, 26 Aug 2024 19:23:11 +0200
Subject: [PATCH 11/14] Batched ledger upgrade

---
 nano/store/lmdb/lmdb.cpp       | 61 +++++++++++++++++++---------
 nano/store/rocksdb/rocksdb.cpp | 73 +++++++++++++++++++++++-----------
 2 files changed, 92 insertions(+), 42 deletions(-)

diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp
index 6c3c5a798b..7e676db0aa 100644
--- a/nano/store/lmdb/lmdb.cpp
+++ b/nano/store/lmdb/lmdb.cpp
@@ -255,33 +255,58 @@ void nano::store::lmdb::component::upgrade_v21_to_v22 (store::write_transaction
 void nano::store::lmdb::component::upgrade_v22_to_v23 (store::write_transaction & transaction)
 {
 	logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23...");
+
 	drop (transaction, tables::rep_weights);
-	auto i{ make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts) };
-	auto end{ store::iterator<nano::account, nano::account_info_v22> (nullptr) };
-	uint64_t processed_accounts = 0;
-	for (; i != end; ++i)
+
+	const size_t batch_size = 1000 * 10;
+
+	nano::account next = 0;
+	size_t processed_accounts = 0;
+	while (true)
 	{
-		if (!i->second.balance.is_zero ())
+		transaction.refresh ();
+
+		// Manually create v22 compatible iterator to read accounts
+		auto it = make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts, next);
+		auto const end = store::iterator<nano::account, nano::account_info_v22> (nullptr);
+
+		if (it == end)
 		{
-			nano::uint128_t total{ 0 };
-			nano::store::lmdb::db_val value;
-			auto status = get (transaction, tables::rep_weights, i->second.representative, value);
-			if (success (status))
-			{
-				total = nano::amount{ value }.number ();
-			}
-			total += i->second.balance.number ();
-			status = put (transaction, tables::rep_weights, i->second.representative, nano::amount{ total });
-			release_assert_success (status);
+			break;
 		}
-		processed_accounts++;
-		if (processed_accounts % 250000 == 0)
+
+		for (size_t count = 0; it != end && count < batch_size; ++it, ++count)
 		{
-			logger.info (nano::log::type::lmdb, "Processed {} accounts", processed_accounts);
+			auto const & account = it->first;
+			auto const & account_info = it->second;
+
+			if (!account_info.balance.is_zero ())
+			{
+				nano::uint128_t total{ 0 };
+				nano::store::lmdb::db_val value;
+				auto status = get (transaction, tables::rep_weights, account_info.representative, value);
+				if (success (status))
+				{
+					total = nano::amount{ value }.number ();
+				}
+				total += account_info.balance.number ();
+				status = put (transaction, tables::rep_weights, account_info.representative, nano::amount{ total });
+				release_assert_success (status);
+			}
+
+			processed_accounts++;
+			if (processed_accounts % 250000 == 0)
+			{
+				logger.info (nano::log::type::lmdb, "Processed {} accounts", processed_accounts);
+			}
+
+			next = account.number () + 1;
 		}
 	}
+
 	logger.info (nano::log::type::lmdb, "Processed {} accounts", processed_accounts);
 	version.put (transaction, 23);
+
 	logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23 completed");
 }
 
diff --git a/nano/store/rocksdb/rocksdb.cpp b/nano/store/rocksdb/rocksdb.cpp
index 2bd4aae5f2..ffda5bb720 100644
--- a/nano/store/rocksdb/rocksdb.cpp
+++ b/nano/store/rocksdb/rocksdb.cpp
@@ -295,38 +295,63 @@ void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transacti
 		drop (transaction, tables::rep_weights);
 	}
 
-	logger.info (nano::log::type::rocksdb, "Creating table rep_weights");
-	::rocksdb::ColumnFamilyOptions new_cf_options;
-	::rocksdb::ColumnFamilyHandle * new_cf_handle;
-	::rocksdb::Status status = db->CreateColumnFamily (new_cf_options, "rep_weights", &new_cf_handle);
-	handles.emplace_back (new_cf_handle);
-
-	auto i{ make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts) };
-	auto end{ store::iterator<nano::account, nano::account_info_v22> (nullptr) };
-	uint64_t processed_accounts = 0;
-	for (; i != end; ++i)
 	{
-		if (!i->second.balance.is_zero ())
+		logger.info (nano::log::type::rocksdb, "Creating table rep_weights");
+		::rocksdb::ColumnFamilyOptions new_cf_options;
+		::rocksdb::ColumnFamilyHandle * new_cf_handle;
+		::rocksdb::Status status = db->CreateColumnFamily (new_cf_options, "rep_weights", &new_cf_handle);
+		handles.emplace_back (new_cf_handle);
+	}
+
+	const size_t batch_size = 1000 * 10;
+
+	nano::account next = 0;
+	size_t processed_accounts = 0;
+	while (true)
+	{
+		transaction.refresh ();
+
+		// Manually create v22 compatible iterator to read accounts
+		auto it = make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts, next);
+		auto const end = store::iterator<nano::account, nano::account_info_v22> (nullptr);
+
+		if (it == end)
 		{
-			nano::uint128_t total{ 0 };
-			nano::store::rocksdb::db_val value;
-			auto status = get (transaction, tables::rep_weights, i->second.representative, value);
-			if (success (status))
-			{
-				total = nano::amount{ value }.number ();
-			}
-			total += i->second.balance.number ();
-			status = put (transaction, tables::rep_weights, i->second.representative, nano::amount{ total });
-			release_assert_success (status);
+			break;
 		}
-		processed_accounts++;
-		if (processed_accounts % 250000 == 0)
+
+		for (size_t count = 0; it != end && count < batch_size; ++it, ++count)
 		{
-			logger.info (nano::log::type::rocksdb, "Processed {} accounts", processed_accounts);
+			auto const & account = it->first;
+			auto const & account_info = it->second;
+
+			if (!account_info.balance.is_zero ())
+			{
+				nano::uint128_t total{ 0 };
+				nano::store::rocksdb::db_val value;
+				auto status = get (transaction, tables::rep_weights, account_info.representative, value);
+				if (success (status))
+				{
+					total = nano::amount{ value }.number ();
+				}
+				total += account_info.balance.number ();
+				status = put (transaction, tables::rep_weights, account_info.representative, nano::amount{ total });
+				release_assert_success (status);
+			}
+
+			processed_accounts++;
+			if (processed_accounts % 250000 == 0)
+			{
+				logger.info (nano::log::type::rocksdb, "Processed {} accounts", processed_accounts);
+			}
+
+			next = account.number () + 1;
 		}
 	}
+
 	logger.info (nano::log::type::rocksdb, "Processed {} accounts", processed_accounts);
 	version.put (transaction, 23);
+
 	logger.info (nano::log::type::rocksdb, "Upgrading database from v22 to v23 completed");
 }
 

From 9762eeb3e0e9d6b4b5684e03da50f74977b88497 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Mon, 26 Aug 2024 20:04:49 +0200
Subject: [PATCH 12/14] Asserts

---
 nano/store/lmdb/lmdb.cpp       | 3 +++
 nano/store/rocksdb/rocksdb.cpp | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp
index 7e676db0aa..46b7ab9c35 100644
--- a/nano/store/lmdb/lmdb.cpp
+++ b/nano/store/lmdb/lmdb.cpp
@@ -257,6 +257,9 @@ void nano::store::lmdb::component::upgrade_v22_to_v23 (store::write_transaction
 	logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23...");
 
 	drop (transaction, tables::rep_weights);
+	transaction.refresh ();
+
+	release_assert (rep_weight.begin (tx_begin_read ()) == rep_weight.end (), "rep weights table must be empty before upgrading to v23");
 
 	const size_t batch_size = 1000 * 10;
 
diff --git a/nano/store/rocksdb/rocksdb.cpp b/nano/store/rocksdb/rocksdb.cpp
index ffda5bb720..148736d201 100644
--- a/nano/store/rocksdb/rocksdb.cpp
+++ b/nano/store/rocksdb/rocksdb.cpp
@@ -293,6 +293,7 @@ void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transacti
 	{
 		logger.info (nano::log::type::rocksdb, "Dropping existing rep_weights table");
 		drop (transaction, tables::rep_weights);
+		transaction.refresh ();
 	}
 
 	{
@@ -301,8 +302,11 @@ void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transacti
 		::rocksdb::ColumnFamilyHandle * new_cf_handle;
 		::rocksdb::Status status = db->CreateColumnFamily (new_cf_options, "rep_weights", &new_cf_handle);
 		handles.emplace_back (new_cf_handle);
+		transaction.refresh ();
 	}
 
+	release_assert (rep_weight.begin (tx_begin_read ()) == rep_weight.end (), "rep weights table must be empty before upgrading to v23");
+
 	const size_t batch_size = 1000 * 10;
 
 	nano::account next = 0;

From e8a8bb8d3c30dd6df7e92ac5ae5c6df94910b12d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?=
 <3044353+pwojcikdev@users.noreply.github.com>
Date: Tue, 27 Aug 2024 10:48:44 +0200
Subject: [PATCH 13/14] Simplify

---
 nano/store/lmdb/lmdb.cpp       | 63 ++++++++++++++++------------------
 nano/store/rocksdb/rocksdb.cpp | 63 ++++++++++++++++------------------
 2 files changed, 60 insertions(+), 66 deletions(-)

diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp
index 46b7ab9c35..a2fbb618ed 100644
--- a/nano/store/lmdb/lmdb.cpp
+++ b/nano/store/lmdb/lmdb.cpp
@@ -261,53 +261,50 @@ void nano::store::lmdb::component::upgrade_v22_to_v23 (store::write_transaction
 
 	release_assert (rep_weight.begin (tx_begin_read ()) == rep_weight.end (), "rep weights table must be empty before upgrading to v23");
 
-	const size_t batch_size = 1000 * 10;
-
-	nano::account next = 0;
-	size_t processed_accounts = 0;
-	while (true)
-	{
-		transaction.refresh ();
+	auto iterate_accounts = [this] (auto && func) {
+		auto transaction = tx_begin_read ();
 
 		// Manually create v22 compatible iterator to read accounts
-		auto it = make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts, next);
+		auto it = make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts);
 		auto const end = store::iterator<nano::account, nano::account_info_v22> (nullptr);
 
-		if (it == end)
-		{
-			break;
-		}
-
-		for (size_t count = 0; it != end && count < batch_size; ++it, ++count)
+		for (; it != end; ++it)
 		{
 			auto const & account = it->first;
 			auto const & account_info = it->second;
 
-			if (!account_info.balance.is_zero ())
-			{
-				nano::uint128_t total{ 0 };
-				nano::store::lmdb::db_val value;
-				auto status = get (transaction, tables::rep_weights, account_info.representative, value);
-				if (success (status))
-				{
-					total = nano::amount{ value }.number ();
-				}
-				total += account_info.balance.number ();
-				status = put (transaction, tables::rep_weights, account_info.representative, nano::amount{ total });
-				release_assert_success (status);
-			}
+			func (account, account_info);
+		}
+	};
+
+	// TODO: Make this smaller in dev builds
+	const size_t batch_size = 250000;
 
-			processed_accounts++;
-			if (processed_accounts % 250000 == 0)
+	size_t processed = 0;
+	iterate_accounts ([this, &transaction, &processed] (nano::account const & account, nano::account_info_v22 const & account_info) {
+		if (!account_info.balance.is_zero ())
+		{
+			nano::uint128_t total{ 0 };
+			nano::store::lmdb::db_val value;
+			auto status = get (transaction, tables::rep_weights, account_info.representative, value);
+			if (success (status))
 			{
-				logger.info (nano::log::type::lmdb, "Processed {} accounts", processed_accounts);
+				total = nano::amount{ value }.number ();
 			}
+			total += account_info.balance.number ();
+			status = put (transaction, tables::rep_weights, account_info.representative, nano::amount{ total });
+			release_assert_success (status);
+		}
 
-			next = account.number () + 1;
+		processed++;
+		if (processed % batch_size == 0)
+		{
+			logger.info (nano::log::type::lmdb, "Processed {} accounts", processed);
+			transaction.refresh (); // Refresh to prevent excessive memory usage
 		}
-	}
+	});
 
-	logger.info (nano::log::type::lmdb, "Processed {} accounts", processed_accounts);
+	logger.info (nano::log::type::lmdb, "Done processing {} accounts", processed);
 	version.put (transaction, 23);
 
 	logger.info (nano::log::type::lmdb, "Upgrading database from v22 to v23 completed");
diff --git a/nano/store/rocksdb/rocksdb.cpp b/nano/store/rocksdb/rocksdb.cpp
index 148736d201..9f89bd8454 100644
--- a/nano/store/rocksdb/rocksdb.cpp
+++ b/nano/store/rocksdb/rocksdb.cpp
@@ -307,53 +307,50 @@ void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transacti
 
 	release_assert (rep_weight.begin (tx_begin_read ()) == rep_weight.end (), "rep weights table must be empty before upgrading to v23");
 
-	const size_t batch_size = 1000 * 10;
-
-	nano::account next = 0;
-	size_t processed_accounts = 0;
-	while (true)
-	{
-		transaction.refresh ();
+	auto iterate_accounts = [this] (auto && func) {
+		auto transaction = tx_begin_read ();
 
 		// Manually create v22 compatible iterator to read accounts
-		auto it = make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts, next);
+		auto it = make_iterator<nano::account, nano::account_info_v22> (transaction, tables::accounts);
 		auto const end = store::iterator<nano::account, nano::account_info_v22> (nullptr);
 
-		if (it == end)
-		{
-			break;
-		}
-
-		for (size_t count = 0; it != end && count < batch_size; ++it, ++count)
+		for (; it != end; ++it)
 		{
 			auto const & account = it->first;
 			auto const & account_info = it->second;
 
-			if (!account_info.balance.is_zero ())
-			{
-				nano::uint128_t total{ 0 };
-				nano::store::rocksdb::db_val value;
-				auto status = get (transaction, tables::rep_weights, account_info.representative, value);
-				if (success (status))
-				{
-					total = nano::amount{ value }.number ();
-				}
-				total += account_info.balance.number ();
-				status = put (transaction, tables::rep_weights, account_info.representative, nano::amount{ total });
-				release_assert_success (status);
-			}
+			func (account, account_info);
+		}
+	};
 
-			processed_accounts++;
-			if (processed_accounts % 250000 == 0)
+	// TODO: Make this smaller in dev builds
+	const size_t batch_size = 250000;
+
+	size_t processed = 0;
+	iterate_accounts ([this, &transaction, &processed] (nano::account const & account, nano::account_info_v22 const & account_info) {
+		if (!account_info.balance.is_zero ())
+		{
+			nano::uint128_t total{ 0 };
+			nano::store::rocksdb::db_val value;
+			auto status = get (transaction, tables::rep_weights, account_info.representative, value);
+			if (success (status))
 			{
-				logger.info (nano::log::type::rocksdb, "Processed {} accounts", processed_accounts);
+				total = nano::amount{ value }.number ();
 			}
+			total += account_info.balance.number ();
+			status = put (transaction, tables::rep_weights, account_info.representative, nano::amount{ total });
+			release_assert_success (status);
+		}
 
-			next = account.number () + 1;
+		processed++;
+		if (processed % batch_size == 0)
+		{
+			logger.info (nano::log::type::rocksdb, "Processed {} accounts", processed);
+			transaction.refresh (); // Refresh to prevent excessive memory usage
 		}
-	}
+	});
 
-	logger.info (nano::log::type::rocksdb, "Processed {} accounts", processed_accounts);
+	logger.info (nano::log::type::rocksdb, "Done processing {} accounts", processed);
 	version.put (transaction, 23);
 
 	logger.info (nano::log::type::rocksdb, "Upgrading database from v22 to v23 completed");

From bea03769e7723e625cd60ee2f88b59395fcfa59b Mon Sep 17 00:00:00 2001
From: Colin LeMahieu <clemahieu@gmail.com>
Date: Tue, 27 Aug 2024 15:11:43 +0100
Subject: [PATCH 14/14] Directly deleting deprecated column_family and checking
 status code for recreating.

---
 nano/store/rocksdb/rocksdb.cpp | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/nano/store/rocksdb/rocksdb.cpp b/nano/store/rocksdb/rocksdb.cpp
index 9f89bd8454..60f7e95cee 100644
--- a/nano/store/rocksdb/rocksdb.cpp
+++ b/nano/store/rocksdb/rocksdb.cpp
@@ -292,7 +292,18 @@ void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transacti
 	if (column_family_exists ("rep_weights"))
 	{
 		logger.info (nano::log::type::rocksdb, "Dropping existing rep_weights table");
-		drop (transaction, tables::rep_weights);
+		auto const rep_weights_handle = get_column_family ("rep_weights");
+		db->DropColumnFamily (rep_weights_handle);
+		db->DestroyColumnFamilyHandle (rep_weights_handle);
+		std::erase_if (handles, [rep_weights_handle] (auto & handle) {
+			if (handle.get () == rep_weights_handle)
+			{
+				// The handle resource is deleted by RocksDB.
+				[[maybe_unused]] auto ptr = handle.release ();
+				return true;
+			}
+			return false;
+		});
 		transaction.refresh ();
 	}
 
@@ -301,6 +312,7 @@ void nano::store::rocksdb::component::upgrade_v22_to_v23 (store::write_transacti
 		::rocksdb::ColumnFamilyOptions new_cf_options;
 		::rocksdb::ColumnFamilyHandle * new_cf_handle;
 		::rocksdb::Status status = db->CreateColumnFamily (new_cf_options, "rep_weights", &new_cf_handle);
+		release_assert (success (status.code ()));
 		handles.emplace_back (new_cf_handle);
 		transaction.refresh ();
 	}
@@ -362,11 +374,11 @@ void nano::store::rocksdb::component::upgrade_v23_to_v24 (store::write_transacti
 
 	if (column_family_exists ("frontiers"))
 	{
-		auto const unchecked_handle = get_column_family ("frontiers");
-		db->DropColumnFamily (unchecked_handle);
-		db->DestroyColumnFamilyHandle (unchecked_handle);
-		std::erase_if (handles, [unchecked_handle] (auto & handle) {
-			if (handle.get () == unchecked_handle)
+		auto const frontiers_handle = get_column_family ("frontiers");
+		db->DropColumnFamily (frontiers_handle);
+		db->DestroyColumnFamilyHandle (frontiers_handle);
+		std::erase_if (handles, [frontiers_handle] (auto & handle) {
+			if (handle.get () == frontiers_handle)
 			{
 				// The handle resource is deleted by RocksDB.
 				[[maybe_unused]] auto ptr = handle.release ();