Skip to content

Commit

Permalink
Support tombstone'd value states in the storage-layer (#161)
Browse files Browse the repository at this point in the history 
* Adjust the public crate-level types which are commonly utilized by clients (not storage implementers)

* Move AkdValue and AkdLabel to binary vectors instead of strings

* Identify tombstones in client & add flag to support them, verification of hash coming

* Tombstoning at the storage layer

* Verify the hash of the plaintext value compared to the hash in the existence proof

* Adding test coverage around tombstoned entries in the key history.

* Debugging statements and rustfmt + clippy

* CI clippy

* Cleanup of leaf hashing logic

* PR review

* Clippy still failing client-side on Mac, so fixing CI indentified clippy lints

* merge clippy's

* rustfmt after clippy :sad_eyes:

* PR review: move tombstone to empty array of data and add more test coverage

* Addressing: #170. Adding a top N key updates to the history call.

Also forcing key history proofs to be in max -> min, such that they go from highest version to lowest

* clippy lint caught in CI

Co-authored-by: Sean Lawlor <[email protected]>
  • Loading branch information
@slawlor
slawlor and slawlor authored Mar 15, 2022
1 parent c7fc085 commit 3c5b89f
Show file tree
Hide file tree
Showing 27 changed files with 1,439 additions and 621 deletions.
4 changes: 1 addition & 3 deletions .vscode/launch.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@
"test",
"--no-run",
"--lib",
"--package=akd_client",
"--features=sha3_256,nostd",
"--no-default-features"
"--package=akd_client"
],
},
"args": [],
Expand Down
39 changes: 20 additions & 19 deletions akd/src/append_only_zks.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ use async_recursion::async_recursion;
use log::{debug, info};
use std::marker::{Send, Sync};
use tokio::time::Instant;
use winter_crypto::{Digest, Hasher};
use winter_crypto::Hasher;

use keyed_priority_queue::{Entry, KeyedPriorityQueue};
use std::collections::HashSet;
Expand Down Expand Up @@ -99,15 +99,15 @@ impl Azks {
let new_leaf = get_leaf_node::<H, S>(
storage,
node.label,
node.hash.as_bytes().as_ref(),
&node.hash,
NodeLabel::root(),
self.latest_epoch,
)
.await?;

let mut root_node = HistoryTreeNode::get_from_storage(
storage,
NodeKey(NodeLabel::root()),
&NodeKey(NodeLabel::root()),
self.get_latest_epoch(),
)
.await?;
Expand Down Expand Up @@ -156,7 +156,7 @@ impl Azks {
while !current_nodes.is_empty() {
let nodes = HistoryTreeNode::batch_get_from_storage(
storage,
current_nodes,
&current_nodes,
self.get_latest_epoch(),
)
.await?;
Expand All @@ -169,7 +169,7 @@ impl Azks {
for node in &nodes {
node_states.push(get_state_map_key(node, node.get_latest_epoch()));
}
let states = storage.batch_get::<HistoryNodeState>(node_states).await?;
let states = storage.batch_get::<HistoryNodeState>(&node_states).await?;
load_count += states.len() as u64;

// Now that states are loaded in the cache, you can read and access them.
Expand Down Expand Up @@ -220,13 +220,12 @@ impl Azks {
);

self.increment_epoch();
self.preload_nodes_for_insertion::<S, H>(storage, &insertion_set)
.await?;

let mut hash_q = KeyedPriorityQueue::<NodeLabel, i32>::new();
let mut priorities: i32 = 0;
let mut root_node = HistoryTreeNode::get_from_storage(
storage,
NodeKey(NodeLabel::root()),
&NodeKey(NodeLabel::root()),
self.get_latest_epoch(),
)
.await?;
Expand All @@ -243,7 +242,7 @@ impl Azks {
get_leaf_node::<H, S>(
storage,
node.label,
node.hash.as_bytes().as_ref(),
&node.hash,
NodeLabel::root(),
self.latest_epoch,
)
Expand All @@ -264,7 +263,7 @@ impl Azks {
while let Some((next_node_label, _)) = hash_q.pop() {
let mut next_node: HistoryTreeNode = HistoryTreeNode::get_from_storage(
storage,
NodeKey(next_node_label),
&NodeKey(next_node_label),
self.get_latest_epoch(),
)
.await?;
Expand Down Expand Up @@ -317,7 +316,7 @@ impl Azks {
.await?;
let lcp_node: HistoryTreeNode = HistoryTreeNode::get_from_storage(
storage,
NodeKey(lcp_node_label),
&NodeKey(lcp_node_label),
self.get_latest_epoch(),
)
.await?;
Expand All @@ -331,16 +330,17 @@ impl Azks {
for (i, child) in state.child_states.iter().enumerate() {
match child {
None => {
println!("i = {}, empty", i);
debug!("i = {}, empty", i);
continue;
}
Some(child) => {
let unwrapped_child: HistoryTreeNode = HistoryTreeNode::get_from_storage(
storage,
NodeKey(child.label),
&NodeKey(child.label),
self.get_latest_epoch(),
)
.await?;
debug!("Label of child {} is {:?}", i, unwrapped_child.label);
longest_prefix_children[i] = Node {
label: unwrapped_child.label,
hash: unwrapped_child
Expand All @@ -350,6 +350,7 @@ impl Azks {
}
}
}
debug!("Lcp label = {:?}", longest_prefix);
Ok(NonMembershipProof {
label,
longest_prefix,
Expand Down Expand Up @@ -378,7 +379,7 @@ impl Azks {
// between these epochs.
let node = HistoryTreeNode::get_from_storage(
storage,
NodeKey(NodeLabel::root()),
&NodeKey(NodeLabel::root()),
self.get_latest_epoch(),
)
.await?;
Expand Down Expand Up @@ -441,7 +442,7 @@ impl Azks {
Some(child_node_state) => {
let child_node = HistoryTreeNode::get_from_storage(
storage,
NodeKey(child_node_state.label),
&NodeKey(child_node_state.label),
self.get_latest_epoch(),
)
.await?;
Expand Down Expand Up @@ -488,7 +489,7 @@ impl Azks {
}
let root_node: HistoryTreeNode = HistoryTreeNode::get_from_storage(
storage,
NodeKey(NodeLabel::root()),
&NodeKey(NodeLabel::root()),
self.get_latest_epoch(),
)
.await?;
Expand Down Expand Up @@ -518,7 +519,7 @@ impl Azks {
let mut layer_proofs = Vec::new();
let mut curr_node: HistoryTreeNode = HistoryTreeNode::get_from_storage(
storage,
NodeKey(NodeLabel::root()),
&NodeKey(NodeLabel::root()),
self.get_latest_epoch(),
)
.await?;
Expand Down Expand Up @@ -561,7 +562,7 @@ impl Azks {
});
let new_curr_node: HistoryTreeNode = HistoryTreeNode::get_from_storage(
storage,
NodeKey(
&NodeKey(
curr_node
.get_child_label_at_epoch::<_, H>(storage, epoch, dir)
.await?,
Expand All @@ -576,7 +577,7 @@ impl Azks {
if !equal {
let new_curr_node: HistoryTreeNode = HistoryTreeNode::get_from_storage(
storage,
NodeKey(prev_node),
&NodeKey(prev_node),
self.get_latest_epoch(),
)
.await?;
Expand Down
2 changes: 1 addition & 1 deletion akd/src/auditor.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ use std::marker::{Send, Sync};
use winter_crypto::Hasher;

use crate::{
append_only_zks::Azks,
errors::{AkdError, AzksError},
proof_structs::AppendOnlyProof,
storage::memory::AsyncInMemoryDatabase,
Azks,
};

/// Verifies an audit proof, given start and end hashes for a merkle patricia tree.
Expand Down
54 changes: 47 additions & 7 deletions akd/src/client.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,19 @@ pub fn lookup_verify<H: Hasher>(
akd_key: AkdLabel,
proof: LookupProof<H>,
) -> Result<(), AkdError> {
let _plaintext_value = proof.plaintext_value;
let version = proof.version;

let marker_version = 1 << get_marker_version(version);
let existence_proof = proof.existence_proof;
let marker_proof = proof.marker_proof;
let freshness_proof = proof.freshness_proof;

if hash_plaintext_value::<H>(&proof.plaintext_value) != existence_proof.hash_val {
return Err(AkdError::Directory(DirectoryError::VerifyLookupProof(
"Hash of plaintext value did not match expected hash in existence proof".to_string(),
)));
}

let fresh_label = existence_proof.label;
vrf_pk.verify_label::<H>(
&akd_key,
Expand Down Expand Up @@ -148,25 +153,32 @@ pub fn lookup_verify<H: Hasher>(
}

/// Verifies a key history proof, given the corresponding sequence of hashes.
/// Returns a vector of whether the validity of a hash could be verified.
/// When false, the value <=> hash validity at the position could not be
/// verified because the value has been removed ("tombstoned") from the storage layer.
pub fn key_history_verify<H: Hasher>(
vrf_pk: &VRFPublicKey,
root_hashes: Vec<H::Digest>,
previous_root_hashes: Vec<Option<H::Digest>>,
uname: AkdLabel,
proof: HistoryProof<H>,
) -> Result<(), AkdError> {
allow_tombstones: bool,
) -> Result<Vec<bool>, AkdError> {
let mut tombstones = vec![];
for (count, update_proof) in proof.proofs.into_iter().enumerate() {
let root_hash = root_hashes[count];
let previous_root_hash = previous_root_hashes[count];
verify_single_update_proof::<H>(
let is_tombstone = verify_single_update_proof::<H>(
root_hash,
vrf_pk,
previous_root_hash,
update_proof,
&uname,
allow_tombstones,
)?;
tombstones.push(is_tombstone);
}
Ok(())
Ok(tombstones)
}

/// Verifies a single update proof
Expand All @@ -176,9 +188,9 @@ fn verify_single_update_proof<H: Hasher>(
previous_root_hash: Option<H::Digest>,
proof: UpdateProof<H>,
uname: &AkdLabel,
) -> Result<(), AkdError> {
allow_tombstones: bool,
) -> Result<bool, AkdError> {
let epoch = proof.epoch;
let _plaintext_value = &proof.plaintext_value;
let version = proof.version;

let existence_vrf_proof = proof.existence_vrf_proof;
Expand All @@ -190,6 +202,27 @@ fn verify_single_update_proof<H: Hasher>(

let non_existence_before_ep = &proof.non_existence_before_ep;

let (is_tombstone, value_hash_valid) = match (allow_tombstones, &proof.plaintext_value) {
(true, bytes) if bytes.0 == crate::TOMBSTONE => {
// A tombstone was encountered, we need to just take the
// hash of the value at "face value" since we don't have
// the real value available
(true, true)
}
(_, bytes) => {
// No tombstone so hash the value found, and compare to the existence proof's value
(
false,
hash_plaintext_value::<H>(bytes) == existence_at_ep.hash_val,
)
}
};
if !value_hash_valid {
return Err(AkdError::Directory(DirectoryError::VerifyKeyHistoryProof(
format!("Hash of plaintext value (v: {}) did not match expected hash in existence proof at epoch {}", version, epoch),
)));
}

// ***** PART 1 ***************************
// Verify the VRF and membership proof for the corresponding label for the version being updated to.
vrf_pk.verify_label::<H>(
Expand Down Expand Up @@ -289,7 +322,9 @@ fn verify_single_update_proof<H: Hasher>(
}
}

Ok(())
// return indicator of if the value <=> hash mapping was verified
// or if the hash was simply taken at face-value. True = hash mapping verified
Ok(is_tombstone)
}

/// Hashes all the children of a node, as well as their labels
Expand All @@ -316,3 +351,8 @@ fn hash_layer<H: Hasher>(hashes: Vec<H::Digest>, parent_label: NodeLabel) -> H::
new_hash = H::merge(&[new_hash, hash_label::<H>(parent_label)]);
new_hash
}

fn hash_plaintext_value<H: Hasher>(value: &crate::AkdValue) -> H::Digest {
let single_hash = crate::utils::value_to_bytes::<H>(value);
H::merge(&[H::hash(&EMPTY_VALUE), single_hash])
}
Loading

0 comments on commit 3c5b89f

Please sign in to comment.