Add extended ledger index to map send blocks to receive blocks

The index is automatically backfilled from existing ledger data on initialization and kept consistent via ledger put/delete block hooks.

Author: Exxenoz

nano/secure/ext_ledger.cpp

@@ -5,7 +5,9 @@
 #include <nano/secure/common.hpp>
 #include <nano/secure/ext_ledger.hpp>
 #include <nano/secure/ledger.hpp>
+#include <nano/store/ext_ledger/receive_block_by_send_block.hpp>
 #include <nano/store/ext_ledger_store.hpp>
+#include <nano/store/ledger/block.hpp>
 #include <nano/store/ledger_store.hpp>
 #include <nano/store/meta.hpp>
 
@@ -36,11 +38,81 @@ void nano::ext_ledger::initialize ()
 
 	if (store.get_mode () != nano::store::open_mode::read_only)
 	{
+		auto has_flags = [&store] (nano::store::ext_ledger_flags flags) -> bool {
+			return store.ext.has_flags (store.tx_begin_read (), flags);
+		};
+
+		if (!has_flags (nano::store::ext_ledger_flags::receive_block_by_send_block_initialized))
+		{
+			store.ext.receive_block_by_send_block.clear ();
+			initialize_receive_block_by_send_block_index ();
+		}
 	}
 
 	initialized = true;
 }
 
+void nano::ext_ledger::initialize_receive_block_by_send_block_index ()
+{
+	nano::store::ledger_store & store = ledger.store;
+	nano::store::write_transaction txn = store.tx_begin_write ();
+
+	release_assert (store.ext.is_initialized (), "Extended ledger store must be initialized");
+	release_assert (store.ext.receive_block_by_send_block.empty (txn), "The index must be cleared before rebuilding to avoid inconsistent or duplicate entries.");
+	release_assert (store.get_mode () != nano::store::open_mode::read_only, "The index cannot be built while the backend is opened in read-only mode.");
+
+	logger.info (nano::log::type::ext_ledger, "Building receive block by send block index from existing ledger data, this may take a while...");
+
+	size_t processed = 0;
+	size_t indexed = 0;
+	size_t const batch_size = 100000;
+
+	for (auto itr = store.block.begin (txn), end = store.block.end (txn); itr != end; ++itr)
+	{
+		auto const & sideband = itr->second;
+		auto block = sideband.block;
+		if (block->is_receive ())
+		{
+			nano::block_hash receive_block_hash = itr->first;
+			nano::block_hash send_block_hash = block->source ();
+			store.ext.receive_block_by_send_block.put (txn, send_block_hash, receive_block_hash);
+			++indexed;
+		}
+		++processed;
+		if (processed % batch_size == 0)
+		{
+			logger.info (nano::log::type::ext_ledger, "Build progress: processed {} blocks, indexed {} entries", processed, indexed);
+
+			txn.refresh ();
+		}
+	}
+
+	logger.info (nano::log::type::ext_ledger, "Build completed: processed {} blocks, indexed {} entries", processed, indexed);
+
+	// Mark index as fully built and consistent with the current ledger state
+	store.ext.add_flags (txn, nano::store::ext_ledger_flags::receive_block_by_send_block_initialized);
+}
+
+void nano::ext_ledger::on_put_block (nano::store::write_transaction const & txn, nano::block_hash const & hash, nano::block const & block)
+{
+	nano::store::ledger_store & store = ledger.store;
+	release_assert (store.ext.is_initialized (), "Extended ledger store must be initialized");
+	if (block.is_receive ())
+	{
+		store.ext.receive_block_by_send_block.put (txn, block.source (), hash);
+	}
+}
+
+void nano::ext_ledger::on_del_block (nano::store::write_transaction const & txn, nano::block_hash const & hash, nano::block const & block)
+{
+	nano::store::ledger_store & store = ledger.store;
+	release_assert (store.ext.is_initialized (), "Extended ledger store must be initialized");
+	if (block.is_receive ())
+	{
+		store.ext.receive_block_by_send_block.del (txn, block.source ());
+	}
+}
+
 void nano::ext_ledger::clear (nano::store::write_transaction const & txn)
 {
 	nano::store::ledger_store & store = ledger.store;

nano/secure/ext_ledger.hpp

@@ -17,6 +17,11 @@ class ext_ledger final
 
 	bool is_initialized ();
 	void initialize ();
+	void initialize_receive_block_by_send_block_index ();
+
+	void on_put_block (nano::store::write_transaction const &, nano::block_hash const &, nano::block const &);
+	void on_del_block (nano::store::write_transaction const &, nano::block_hash const &, nano::block const &);
+
 	void clear (nano::store::write_transaction const &);
 	void drop (nano::store::write_transaction const &);
 

nano/secure/ledger.cpp

@@ -16,6 +16,7 @@
 #include <nano/secure/ledger_set_any.hpp>
 #include <nano/secure/ledger_set_confirmed.hpp>
 #include <nano/secure/rep_weights.hpp>
+#include <nano/store/ext_ledger/receive_block_by_send_block.hpp>
 #include <nano/store/ext_ledger_store.hpp>
 #include <nano/store/ledger/account.hpp>
 #include <nano/store/ledger/block.hpp>
@@ -79,10 +80,22 @@ void nano::ledger::del_account (nano::store::write_transaction const & transacti
 void nano::ledger::put_block (nano::store::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block const & block_a)
 {
 	store.block.put (transaction_a, hash_a, block_a);
+
+	if (ext.is_initialized ())
+	{
+		ext.on_put_block (transaction_a, hash_a, block_a);
+	}
 }
 
 void nano::ledger::del_block (nano::store::write_transaction const & transaction_a, nano::block_hash const & hash_a)
 {
+	if (ext.is_initialized ())
+	{
+		auto block = store.block.get (transaction_a, hash_a);
+		release_assert (block, "Block to be deleted was not found in the ledger");
+		ext.on_del_block (transaction_a, hash_a, *block);
+	}
+
 	store.block.del (transaction_a, hash_a);
 }
 
@@ -724,6 +737,17 @@ std::shared_ptr<nano::block> nano::ledger::find_receive_block_by_send_hash (secu
 	std::shared_ptr<nano::block> result;
 	debug_assert (send_block_hash != 0);
 
+	// Use the extended ledger index to directly look up the receive block by send block hash, if enabled
+	if (store.ext.is_initialized ())
+	{
+		auto receive_block_hash = store.ext.receive_block_by_send_block.get (transaction, send_block_hash);
+		if (receive_block_hash.has_value ())
+		{
+			return store.block.get (transaction, receive_block_hash.value ());
+		}
+		return nullptr;
+	}
+
 	// get the cemented frontier
 	nano::confirmation_height_info info;
 	if (store.confirmation_height.get (transaction, destination, info))

nano/store/CMakeLists.txt

@@ -6,6 +6,8 @@ add_library(
   ext_ledger_store.cpp
   backend.hpp
   backend.cpp
+  ext_ledger/receive_block_by_send_block.hpp
+  ext_ledger/receive_block_by_send_block.cpp
   ledger/account.hpp
   ledger/account.cpp
   ledger/block.hpp

nano/store/ext_ledger/receive_block_by_send_block.cpp

@@ -0,0 +1,45 @@
+#include <nano/store/db_val_templ.hpp>
+#include <nano/store/ext_ledger/receive_block_by_send_block.hpp>
+
+namespace nano::store::ext_ledger
+{
+receive_block_by_send_block_view::receive_block_by_send_block_view (nano::store::backend & backend_a) :
+	backend{ backend_a }
+{
+}
+
+void receive_block_by_send_block_view::put (nano::store::write_transaction const & txn, nano::block_hash const & send_block_hash, nano::block_hash const & receive_block_hash)
+{
+	auto status = backend.put (txn, nano::store::table::ext_receive_block_by_send_block, send_block_hash, receive_block_hash);
+	backend.release_assert_success (status);
+}
+
+std::optional<nano::block_hash> receive_block_by_send_block_view::get (nano::store::transaction const & txn, nano::block_hash const & send_block_hash) const
+{
+	nano::store::db_val result;
+	auto status = backend.get (txn, nano::store::table::ext_receive_block_by_send_block, send_block_hash, result);
+	std::optional<nano::block_hash> receive_block_hash;
+	if (backend.success (status))
+	{
+		receive_block_hash = static_cast<nano::block_hash> (result);
+	}
+	return receive_block_hash;
+}
+
+void receive_block_by_send_block_view::del (nano::store::write_transaction const & txn, nano::block_hash const & send_block_hash)
+{
+	auto status = backend.del (txn, nano::store::table::ext_receive_block_by_send_block, send_block_hash);
+	backend.release_assert_success (status);
+}
+
+bool receive_block_by_send_block_view::empty (nano::store::transaction const & txn) const
+{
+	return backend.empty (txn, nano::store::table::ext_receive_block_by_send_block);
+}
+
+void receive_block_by_send_block_view::clear ()
+{
+	auto status = backend.clear (nano::store::table::ext_receive_block_by_send_block);
+	backend.release_assert_success (status);
+}
+}

nano/store/ext_ledger/receive_block_by_send_block.hpp

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <nano/lib/numbers.hpp>
+#include <nano/store/backend.hpp>
+
+#include <functional>
+
+namespace nano::store::ext_ledger
+{
+class receive_block_by_send_block_view
+{
+public:
+	explicit receive_block_by_send_block_view (nano::store::backend &);
+
+	void put (nano::store::write_transaction const &, nano::block_hash const &, nano::block_hash const &);
+	std::optional<nano::block_hash> get (nano::store::transaction const &, nano::block_hash const &) const;
+	void del (nano::store::write_transaction const &, nano::block_hash const &);
+	bool empty (nano::store::transaction const &) const;
+	void clear ();
+
+private:
+	nano::store::backend & backend;
+};
+}

nano/store/ext_ledger_store.cpp

@@ -1,12 +1,14 @@
 #include <nano/lib/logging.hpp>
 #include <nano/lib/stats.hpp>
 #include <nano/store/backend.hpp>
+#include <nano/store/ext_ledger/receive_block_by_send_block.hpp>
 #include <nano/store/ext_ledger_store.hpp>
 #include <nano/store/ledger_store.hpp>
 
 namespace nano::store
 {
 nano::store::column_schema const ext_ledger_store::schema_current{
+	{ nano::store::table::ext_receive_block_by_send_block, "ext_receive_block_by_send_block" },
 	{ nano::store::table::meta, "meta" }
 };
 }
@@ -18,7 +20,9 @@ ext_ledger_store::ext_ledger_store (nano::store::backend & backend_a, nano::stat
 	stats{ stats_a },
 	logger{ logger_a },
 	meta_impl{ std::make_unique<nano::store::meta_view> (backend_a) },
-	meta{ *meta_impl }
+	receive_block_by_send_block_impl{ std::make_unique<nano::store::ext_ledger::receive_block_by_send_block_view> (backend_a) },
+	meta{ *meta_impl },
+	receive_block_by_send_block{ *receive_block_by_send_block_impl }
 {
 }
 

nano/store/ext_ledger_store.hpp

@@ -14,6 +14,7 @@ namespace nano::store
 enum class ext_ledger_flags : uint64_t
 {
 	none = 0,
+	receive_block_by_send_block_initialized = 1 << 0,
 };
 
 inline nano::store::ext_ledger_flags operator| (nano::store::ext_ledger_flags a, nano::store::ext_ledger_flags b)
@@ -55,9 +56,11 @@ class ext_ledger_store
 
 private:
 	std::unique_ptr<nano::store::meta_view> meta_impl;
+	std::unique_ptr<nano::store::ext_ledger::receive_block_by_send_block_view> receive_block_by_send_block_impl;
 
 public:
 	nano::store::meta_view & meta;
+	nano::store::ext_ledger::receive_block_by_send_block_view & receive_block_by_send_block;
 
 public:
 	static nano::store::version_value_t constexpr version_minimum{ 1 };

nano/store/fwd.hpp

@@ -14,6 +14,11 @@ class transaction;
 class write_transaction;
 }
 
+namespace nano::store::ext_ledger
+{
+class receive_block_by_send_block_view;
+}
+
 namespace nano::store::ledger
 {
 class account_view;

nano/store/rocksdb/backend_rocksdb.cpp

@@ -353,6 +353,7 @@ bool backend_rocksdb::count_is_exact (nano::store::table table) const
 		case nano::store::table::blocks:
 		case nano::store::table::confirmation_height:
 		case nano::store::table::default_unused:
+		case nano::store::table::ext_receive_block_by_send_block:
 		case nano::store::table::meta:
 		case nano::store::table::online_weight:
 		case nano::store::table::peers: