Change missing_nodes to accept hypercore index

Add missing_nodes_from_merkle_tree_index for a possibility to evaluate also odd merkle tree indices.
Also add replication example.

Author: ttiurani

Cargo.toml

@@ -1,12 +1,15 @@
 [package]
 name = "hypercore"
-version = "0.12.0-alpha.7"
+version = "0.12.0-alpha.8"
 license = "MIT OR Apache-2.0"
 description = "Secure, distributed, append-only log"
 documentation = "https://docs.rs/hypercore"
 repository = "https://github.com/datrs/hypercore"
 readme = "README.md"
-authors = ["Yoshua Wuyts <[email protected]>"]
+authors = [
+  "Yoshua Wuyts <[email protected]>",
+  "Timo Tiuraniemi <[email protected]>"
+]
 keywords = ["dat", "p2p", "stream", "feed", "merkle"]
 categories = [
   "asynchronous",

examples/disk.rs

@@ -1,10 +1,11 @@
 #[cfg(feature = "async-std")]
 use async_std::main as async_main;
-use hypercore::{HypercoreBuilder, Storage};
+use hypercore::{HypercoreBuilder, HypercoreError, Storage};
 use tempfile::Builder;
 #[cfg(feature = "tokio")]
 use tokio::main as async_main;
 
+/// Example about using an in-memory hypercore.
 #[async_main]
 async fn main() {
     // For the purposes of this example, first create a
@@ -72,8 +73,16 @@ async fn main() {
     // Print the first three values, converting binary back to string
     println!(
         "{}{}{}",
-        String::from_utf8(hypercore.get(0).await.unwrap().unwrap()).unwrap(),
-        String::from_utf8(hypercore.get(1).await.unwrap().unwrap()).unwrap(),
-        String::from_utf8(hypercore.get(2).await.unwrap().unwrap()).unwrap()
+        format_res(hypercore.get(0).await),
+        format_res(hypercore.get(1).await),
+        format_res(hypercore.get(2).await)
     ); // prints "Hello, from disk hypercore!"
 }
+
+fn format_res(res: Result<Option<Vec<u8>>, HypercoreError>) -> String {
+    match res {
+        Ok(Some(bytes)) => String::from_utf8(bytes).expect("Shouldn't fail in example"),
+        Ok(None) => "Got None in feed".to_string(),
+        Err(e) => format!("Error getting value from feed, reason = {e:?}"),
+    }
+}

examples/memory.rs

@@ -1,9 +1,10 @@
 #[cfg(feature = "async-std")]
 use async_std::main as async_main;
-use hypercore::{HypercoreBuilder, Storage};
+use hypercore::{HypercoreBuilder, HypercoreError, Storage};
 #[cfg(feature = "tokio")]
 use tokio::main as async_main;
 
+/// Example about using an in-memory hypercore.
 #[async_main]
 async fn main() {
     // Create a memory storage
@@ -44,7 +45,15 @@ async fn main() {
     // Print values, converting binary back to string
     println!(
         "{}{}",
-        String::from_utf8(hypercore.get(0).await.unwrap().unwrap()).unwrap(),
-        String::from_utf8(hypercore.get(1).await.unwrap().unwrap()).unwrap()
+        format_res(hypercore.get(0).await),
+        format_res(hypercore.get(1).await)
     ); // prints "Hello, from memory hypercore!"
 }
+
+fn format_res(res: Result<Option<Vec<u8>>, HypercoreError>) -> String {
+    match res {
+        Ok(Some(bytes)) => String::from_utf8(bytes).expect("Shouldn't fail in example"),
+        Ok(None) => "Got None in feed".to_string(),
+        Err(e) => format!("Error getting value from feed, reason = {e:?}"),
+    }
+}

examples/replication.rs

@@ -0,0 +1,116 @@
+#[cfg(feature = "async-std")]
+use async_std::main as async_main;
+use hypercore::{
+    Hypercore, HypercoreBuilder, HypercoreError, PartialKeypair, RequestBlock, RequestUpgrade,
+    Storage,
+};
+use random_access_disk::RandomAccessDisk;
+use random_access_memory::RandomAccessMemory;
+use tempfile::Builder;
+#[cfg(feature = "tokio")]
+use tokio::main as async_main;
+
+/// Example on how to replicate a (disk) hypercore to another (memory) hypercore.
+/// NB: The replication functions used here are low-level, built for use in the wire
+/// protocol.
+#[async_main]
+async fn main() {
+    // For the purposes of this example, first create a
+    // temporary directory to hold hypercore.
+    let dir = Builder::new()
+        .prefix("examples_replication")
+        .tempdir()
+        .unwrap()
+        .into_path();
+
+    // Create a disk storage, overwriting existing values.
+    let overwrite = true;
+    let storage = Storage::new_disk(&dir, overwrite)
+        .await
+        .expect("Could not create disk storage");
+
+    // Build a new disk hypercore
+    let mut origin_hypercore = HypercoreBuilder::new(storage)
+        .build()
+        .await
+        .expect("Could not create disk hypercore");
+
+    // Append values to the hypercore
+    let batch: &[&[u8]] = &[b"Hello, ", b"from ", b"replicated ", b"hypercore!"];
+    origin_hypercore.append_batch(batch).await.unwrap();
+
+    // Store the public key
+    let origin_public_key = origin_hypercore.key_pair().public;
+
+    // Create a peer of the origin hypercore using the public key
+    let mut replicated_hypercore = HypercoreBuilder::new(
+        Storage::new_memory()
+            .await
+            .expect("Could not create memory storage"),
+    )
+    .key_pair(PartialKeypair {
+        public: origin_public_key,
+        secret: None,
+    })
+    .build()
+    .await
+    .expect("Could not create memory hypercore");
+
+    // Replicate the four values in random order
+    replicate_index(&mut origin_hypercore, &mut replicated_hypercore, 3).await;
+    replicate_index(&mut origin_hypercore, &mut replicated_hypercore, 0).await;
+    replicate_index(&mut origin_hypercore, &mut replicated_hypercore, 2).await;
+    replicate_index(&mut origin_hypercore, &mut replicated_hypercore, 1).await;
+
+    // Print values from replicated hypercore, converting binary back to string
+    println!(
+        "{}{}{}{}",
+        format_res(replicated_hypercore.get(0).await),
+        format_res(replicated_hypercore.get(1).await),
+        format_res(replicated_hypercore.get(2).await),
+        format_res(replicated_hypercore.get(3).await)
+    ); // prints "Hello, from replicated hypercore!"
+}
+
+async fn replicate_index(
+    origin_hypercore: &mut Hypercore<RandomAccessDisk>,
+    replicated_hypercore: &mut Hypercore<RandomAccessMemory>,
+    request_index: u64,
+) {
+    let missing_nodes = origin_hypercore
+        .missing_nodes(request_index)
+        .await
+        .expect("Could not get missing nodes");
+    let upgrade_start = replicated_hypercore.info().contiguous_length;
+    let upgrade_length = origin_hypercore.info().contiguous_length - upgrade_start;
+
+    let proof = origin_hypercore
+        .create_proof(
+            Some(RequestBlock {
+                index: request_index,
+                nodes: missing_nodes,
+            }),
+            None,
+            None,
+            Some(RequestUpgrade {
+                start: upgrade_start,
+                length: upgrade_length,
+            }),
+        )
+        .await
+        .expect("Creating proof error")
+        .expect("Could not get proof");
+    // Then the proof is verified and applied to the replicated party.
+    assert!(replicated_hypercore
+        .verify_and_apply_proof(&proof)
+        .await
+        .expect("Verifying and applying proof failed"));
+}
+
+fn format_res(res: Result<Option<Vec<u8>>, HypercoreError>) -> String {
+    match res {
+        Ok(Some(bytes)) => String::from_utf8(bytes).expect("Shouldn't fail in example"),
+        Ok(None) => "Got None in feed".to_string(),
+        Err(e) => format!("Error getting value from feed, reason = {e:?}"),
+    }
+}

src/core.rs

@@ -550,7 +550,17 @@ where
     /// Used to fill the nodes field of a `RequestBlock` during
     /// synchronization.
     #[instrument(err, skip(self))]
-    pub async fn missing_nodes(&mut self, merkle_tree_index: u64) -> Result<u64, HypercoreError> {
+    pub async fn missing_nodes(&mut self, index: u64) -> Result<u64, HypercoreError> {
+        self.missing_nodes_from_merkle_tree_index(index * 2).await
+    }
+
+    /// Get missing nodes using a merkle tree index. Advanced variant of missing_nodex
+    /// that allow for special cases of searching directly from the merkle tree.
+    #[instrument(err, skip(self))]
+    pub async fn missing_nodes_from_merkle_tree_index(
+        &mut self,
+        merkle_tree_index: u64,
+    ) -> Result<u64, HypercoreError> {
         match self.tree.missing_nodes(merkle_tree_index, None)? {
             Either::Right(value) => Ok(value),
             Either::Left(instructions) => {

src/tree/merkle_tree.rs

@@ -328,7 +328,7 @@ impl MerkleTree {
 
     /// Creates valueless proof from requests.
     /// TODO: This is now just a clone of javascript's
-    /// https://github.com/hypercore-protocol/hypercore/blob/7e30a0fe353c70ada105840ec1ead6627ff521e7/lib/merkle-tree.js#L604
+    /// https://github.com/holepunchto/hypercore/blob/9ce03363cb8938dbab53baba7d7cc9dde0508a7e/lib/merkle-tree.js#L1181
     /// The implementation should be rewritten to make it clearer.
     pub(crate) fn create_valueless_proof(
         &mut self,