lightning-block-sync: Implement ser. logic for header Cache types

During syncing, `lightning-block-sync` populates as a block header
`Cache` that is crucial to be able to disconnected previously-connected
headers cleanly in case of a reorg. Moreover, the `Cache` can have
performance benefits as subsequently synced listeners might not
necessarily need to lookup all headers again from the chain source.

While this `Cache` is ~crucial to the clean operation of
`lightning-block-sync`, it was previously not possible to persist it to
disk due to an absence of serialization logic implementations for the
corresponding sub-types. Here, we do just that (Implement said
serialization logic) to allow users to persist the `Cache`.

Making use of the serialization logic for all the sub-types, we also
switch the `UnboundedCache` type to be a newtype wrapper around a
`HashMap` (rather than a straight typedef) and implement TLV-based
serialization logic on it.

lightning-block-sync/Cargo.toml

@@ -20,6 +20,7 @@ rpc-client = [ "serde_json", "chunked_transfer" ]
 [dependencies]
 bitcoin = "0.32.2"
 lightning = { version = "0.2.0", path = "../lightning" }
+lightning-macros = { version = "0.2", path = "../lightning-macros" }
 tokio = { version = "1.35", features = [ "io-util", "net", "time", "rt" ], optional = true }
 serde_json = { version = "1.0", optional = true }
 chunked_transfer = { version = "1.4", optional = true }

lightning-block-sync/src/init.rs

@@ -321,8 +321,8 @@ mod tests {
 			(fork_chain_3.tip().block_hash, &listener_3 as &dyn chain::Listen),
 		];
 		let mut cache = fork_chain_1.header_cache(2..=4);
-		cache.extend(fork_chain_2.header_cache(3..=4));
-		cache.extend(fork_chain_3.header_cache(4..=4));
+		cache.inner.extend(fork_chain_2.header_cache(3..=4).inner);
+		cache.inner.extend(fork_chain_3.header_cache(4..=4).inner);
 		match synchronize_listeners(&main_chain, Network::Bitcoin, &mut cache, listeners).await {
 			Ok(header) => assert_eq!(header, main_chain.tip()),
 			Err(e) => panic!("Unexpected error: {:?}", e),
@@ -364,8 +364,8 @@ mod tests {
 			(fork_chain_3.tip().block_hash, &listener_3 as &dyn chain::Listen),
 		];
 		let mut cache = fork_chain_1.header_cache(2..=4);
-		cache.extend(fork_chain_2.header_cache(3..=4));
-		cache.extend(fork_chain_3.header_cache(4..=4));
+		cache.inner.extend(fork_chain_2.header_cache(3..=4).inner);
+		cache.inner.extend(fork_chain_3.header_cache(4..=4).inner);
 		match synchronize_listeners(&main_chain, Network::Bitcoin, &mut cache, listeners).await {
 			Ok(header) => assert_eq!(header, main_chain.tip()),
 			Err(e) => panic!("Unexpected error: {:?}", e),
@@ -387,8 +387,8 @@ mod tests {
 		let mut cache = fork_chain.header_cache(2..=2);
 		match synchronize_listeners(&main_chain, Network::Bitcoin, &mut cache, listeners).await {
 			Ok(_) => {
-				assert!(cache.contains_key(&new_tip.block_hash));
-				assert!(cache.contains_key(&old_tip.block_hash));
+				assert!(cache.inner.contains_key(&new_tip.block_hash));
+				assert!(cache.inner.contains_key(&old_tip.block_hash));
 			},
 			Err(e) => panic!("Unexpected error: {:?}", e),
 		}

lightning-block-sync/src/lib.rs

@@ -48,8 +48,9 @@ use bitcoin::block::{Block, Header};
 use bitcoin::hash_types::BlockHash;
 use bitcoin::pow::Work;
 
-use lightning::chain;
 use lightning::chain::Listen;
+use lightning::util::hash_tables::{new_hash_map, HashMap};
+use lightning::{chain, impl_writeable_tlv_based};
 
 use std::future::Future;
 use std::ops::Deref;
@@ -155,6 +156,12 @@ pub struct BlockHeaderData {
 	pub chainwork: Work,
 }
 
+impl_writeable_tlv_based!(BlockHeaderData, {
+	(0, header, required),
+	(2, height, required),
+	(2, chainwork, required),
+});
+
 /// A block including either all its transactions or only the block header.
 ///
 /// [`BlockSource`] may be implemented to either always return full blocks or, in the case of
@@ -212,22 +219,36 @@ pub trait Cache {
 }
 
 /// Unbounded cache of block headers keyed by block hash.
-pub type UnboundedCache = std::collections::HashMap<BlockHash, ValidatedBlockHeader>;
+pub struct UnboundedCache {
+	pub(crate) inner: HashMap<BlockHash, ValidatedBlockHeader>,
+}
+
+impl UnboundedCache {
+	/// Returns a new `UnboundedCache`
+	pub fn new() -> Self {
+		let inner = new_hash_map();
+		Self { inner }
+	}
+}
 
 impl Cache for UnboundedCache {
 	fn look_up(&self, block_hash: &BlockHash) -> Option<&ValidatedBlockHeader> {
-		self.get(block_hash)
+		self.inner.get(block_hash)
 	}
 
 	fn block_connected(&mut self, block_hash: BlockHash, block_header: ValidatedBlockHeader) {
-		self.insert(block_hash, block_header);
+		self.inner.insert(block_hash, block_header);
 	}
 
 	fn block_disconnected(&mut self, block_hash: &BlockHash) -> Option<ValidatedBlockHeader> {
-		self.remove(block_hash)
+		self.inner.remove(block_hash)
 	}
 }
 
+impl_writeable_tlv_based!(UnboundedCache, {
+	(0, inner, required),
+});
+
 impl<'a, P: Poll, C: Cache, L: Deref> SpvClient<'a, P, C, L>
 where
 	L::Target: chain::Listen,

lightning-block-sync/src/poll.rs

@@ -7,7 +7,9 @@ use crate::{
 
 use bitcoin::hash_types::BlockHash;
 use bitcoin::network::Network;
+
 use lightning::chain::BestBlock;
+use lightning::impl_writeable_tlv_based;
 
 use std::ops::Deref;
 
@@ -171,6 +173,11 @@ impl ValidatedBlockHeader {
 	}
 }
 
+impl_writeable_tlv_based!(ValidatedBlockHeader, {
+	(0, block_hash, required),
+	(2, inner, required),
+});
+
 /// A block with validated data against its transaction list and corresponding block hash.
 pub struct ValidatedBlock {
 	pub(crate) block_hash: BlockHash,

lightning-block-sync/src/test_utils.rs

@@ -131,7 +131,7 @@ impl Blockchain {
 		for i in heights {
 			let value = self.at_height(i);
 			let key = value.header.block_hash();
-			assert!(cache.insert(key, value).is_none());
+			assert!(cache.inner.insert(key, value).is_none());
 		}
 		cache
 	}

lightning/src/util/ser.rs

@@ -24,12 +24,14 @@ use core::ops::Deref;
 use alloc::collections::BTreeMap;
 
 use bitcoin::amount::Amount;
+use bitcoin::block::Header;
 use bitcoin::consensus::Encodable;
 use bitcoin::constants::ChainHash;
 use bitcoin::hash_types::{BlockHash, Txid};
 use bitcoin::hashes::hmac::Hmac;
 use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::hashes::sha256d::Hash as Sha256dHash;
+use bitcoin::pow::Work;
 use bitcoin::script::{self, ScriptBuf};
 use bitcoin::secp256k1::constants::{
 	COMPACT_SIGNATURE_SIZE, PUBLIC_KEY_SIZE, SCHNORR_SIGNATURE_SIZE, SECRET_KEY_SIZE,
@@ -1220,6 +1222,26 @@ impl Readable for Sha256dHash {
 	}
 }
 
+const WORK_SIZE: usize = 32;
+
+impl Writeable for Work {
+	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+		self.to_be_bytes().write(w)
+	}
+
+	#[inline]
+	fn serialized_length(&self) -> usize {
+		WORK_SIZE
+	}
+}
+
+impl Readable for Work {
+	fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+		let buf: [u8; WORK_SIZE] = Readable::read(r)?;
+		Ok(Work::from_be_bytes(buf))
+	}
+}
+
 impl Writeable for ecdsa::Signature {
 	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
 		self.serialize_compact().write(w)
@@ -1429,6 +1451,7 @@ macro_rules! impl_consensus_ser {
 		}
 	};
 }
+impl_consensus_ser!(Header);
 impl_consensus_ser!(Transaction);
 impl_consensus_ser!(TxOut);
 impl_consensus_ser!(Witness);