From 5fe148df5cd5e5f1a6e5d9b65c2990a6d3fe3770 Mon Sep 17 00:00:00 2001 From: ryardley Date: Fri, 11 Oct 2024 15:33:31 +1100 Subject: [PATCH 01/49] Added DataStore struct to act as an injection point for persistence --- packages/ciphernode/Cargo.lock | 1 + packages/ciphernode/data/Cargo.toml | 1 + packages/ciphernode/data/src/data_store.rs | 49 +++++++++++++++++++ .../data/src/{data.rs => in_mem.rs} | 36 +++----------- packages/ciphernode/data/src/lib.rs | 6 ++- .../ciphernode/enclave_node/src/ciphernode.rs | 9 ++-- packages/ciphernode/keyshare/src/keyshare.rs | 14 +++--- packages/ciphernode/router/src/hooks.rs | 4 +- .../tests/test_aggregation_and_decryption.rs | 7 +-- 9 files changed, 80 insertions(+), 47 deletions(-) create mode 100644 packages/ciphernode/data/src/data_store.rs rename packages/ciphernode/data/src/{data.rs => in_mem.rs} (64%) diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index d7492d72..8738f222 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -1841,6 +1841,7 @@ name = "data" version = "0.1.0" dependencies = [ "actix", + "anyhow", ] [[package]] diff --git a/packages/ciphernode/data/Cargo.toml b/packages/ciphernode/data/Cargo.toml index 5624cf4f..a69223fd 100644 --- a/packages/ciphernode/data/Cargo.toml +++ b/packages/ciphernode/data/Cargo.toml @@ -7,3 +7,4 @@ repository = "https://github.com/gnosisguild/enclave/packages/ciphernode" [dependencies] actix = { workspace = true } +anyhow = { workspace = true } diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs new file mode 100644 index 00000000..90b4a5e1 --- /dev/null +++ b/packages/ciphernode/data/src/data_store.rs @@ -0,0 +1,49 @@ +use actix::{Addr, Message, Recipient}; +use anyhow::Result; + +use crate::InMemDataStore; + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] +#[rtype(result = "()")] +pub struct Insert(pub Vec, pub Vec); +impl Insert { + pub fn key(&self) -> Vec { + self.0.clone() + } + + pub fn value(&self) -> Vec { + self.1.clone() + } +} + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] +#[rtype(result = "Option>")] +pub struct Get(pub Vec); +impl Get { + pub fn key(&self) -> Vec { + self.0.clone() + } +} + +#[derive(Clone)] +pub struct DataStore(Recipient, Recipient); +impl DataStore { + pub async fn read(&self, msg: Get) -> Result>> { + Ok(self.0.send(msg).await?) + } + + pub fn write(&self, msg: Insert) { + self.1.do_send(msg) + } + + // use this for testing + pub fn from_in_mem(addr: Addr) -> Self { + Self(addr.clone().recipient(), addr.clone().recipient()) + } + + // // use this for production + // pub fn from_sled(&data_addr: Addr) -> Self { + // let d = data_addr.clone(); + // Self(d.recipient(),d.recipient()) + // } +} diff --git a/packages/ciphernode/data/src/data.rs b/packages/ciphernode/data/src/in_mem.rs similarity index 64% rename from packages/ciphernode/data/src/data.rs rename to packages/ciphernode/data/src/in_mem.rs index 2b6804c8..1aa4993e 100644 --- a/packages/ciphernode/data/src/data.rs +++ b/packages/ciphernode/data/src/in_mem.rs @@ -1,29 +1,7 @@ use actix::{Actor, Context, Handler, Message}; use std::collections::BTreeMap; -// TODO: replace with sled version - -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] -#[rtype(result = "()")] -pub struct Insert(pub Vec, pub Vec); -impl Insert { - fn key(&self) -> Vec { - self.0.clone() - } - - fn value(&self) -> Vec { - self.1.clone() - } -} - -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] -#[rtype(result = "Option>")] -pub struct Get(pub Vec); -impl Get { - fn key(&self) -> Vec { - self.0.clone() - } -} +use crate::{Get, Insert}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "Vec")] @@ -34,17 +12,17 @@ pub enum DataOp { Insert(Insert), } -pub struct Data { +pub struct InMemDataStore { db: BTreeMap, Vec>, log: Vec, capture: bool, } -impl Actor for Data { +impl Actor for InMemDataStore { type Context = Context; } -impl Data { +impl InMemDataStore { pub fn new(capture: bool) -> Self { Self { db: BTreeMap::new(), @@ -54,7 +32,7 @@ impl Data { } } -impl Handler for Data { +impl Handler for InMemDataStore { type Result = (); fn handle(&mut self, event: Insert, _: &mut Self::Context) { // insert data into sled @@ -66,7 +44,7 @@ impl Handler for Data { } } -impl Handler for Data { +impl Handler for InMemDataStore { type Result = Option>; fn handle(&mut self, event: Get, _: &mut Self::Context) -> Option> { let key = event.key(); @@ -74,7 +52,7 @@ impl Handler for Data { } } -impl Handler for Data { +impl Handler for InMemDataStore { type Result = Vec; fn handle(&mut self, _: GetLog, _: &mut Self::Context) -> Vec { self.log.clone() diff --git a/packages/ciphernode/data/src/lib.rs b/packages/ciphernode/data/src/lib.rs index 3ad1dc3a..6acf7a73 100644 --- a/packages/ciphernode/data/src/lib.rs +++ b/packages/ciphernode/data/src/lib.rs @@ -1,2 +1,4 @@ -mod data; -pub use data::*; +mod in_mem; +mod data_store; +pub use in_mem::*; +pub use data_store::*; diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index d2644324..43f59e2c 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -1,7 +1,7 @@ use actix::{Actor, Addr, Context}; use alloy::primitives::Address; use anyhow::Result; -use data::Data; +use data::{DataStore, InMemDataStore}; use enclave_core::EventBus; use evm::{CiphernodeRegistrySol, EnclaveSolReader}; use logger::SimpleLogger; @@ -21,7 +21,7 @@ use crate::app_config::AppConfig; pub struct MainCiphernode { addr: Address, bus: Addr, - data: Addr, + data: DataStore, sortition: Addr, selector: Addr, e3_manager: Addr, @@ -32,7 +32,7 @@ impl MainCiphernode { pub fn new( addr: Address, bus: Addr, - data: Addr, + data: DataStore, sortition: Addr, selector: Addr, p2p: Addr, @@ -57,7 +57,8 @@ impl MainCiphernode { rand_chacha::ChaCha20Rng::from_rng(OsRng).expect("Failed to create RNG"), )); let bus = EventBus::new(true).start(); - let data = Data::new(true).start(); // TODO: Use a sled backed Data Actor + // TODO: switch to Sled actor + let data = DataStore::from_in_mem(InMemDataStore::new(true).start()); let sortition = Sortition::attach(bus.clone()); let selector = CiphernodeSelector::attach(bus.clone(), sortition.clone(), &address.to_string()); diff --git a/packages/ciphernode/keyshare/src/keyshare.rs b/packages/ciphernode/keyshare/src/keyshare.rs index 2d036a97..113b828f 100644 --- a/packages/ciphernode/keyshare/src/keyshare.rs +++ b/packages/ciphernode/keyshare/src/keyshare.rs @@ -1,6 +1,6 @@ use actix::prelude::*; use anyhow::{anyhow, Context, Result}; -use data::{Data, Get, Insert}; +use data::{DataStore, Get, Insert}; use enclave_core::{ CiphernodeSelected, CiphertextOutputPublished, DecryptionshareCreated, Die, EnclaveErrorType, EnclaveEvent, EventBus, FromError, KeyshareCreated, @@ -10,7 +10,7 @@ use std::sync::Arc; pub struct Keyshare { fhe: Arc, - data: Addr, + data: DataStore, bus: Addr, address: String, } @@ -20,7 +20,7 @@ impl Actor for Keyshare { } impl Keyshare { - pub fn new(bus: Addr, data: Addr, fhe: Arc, address: &str) -> Self { + pub fn new(bus: Addr, data: DataStore, fhe: Arc, address: &str) -> Self { Self { bus, fhe, @@ -64,11 +64,11 @@ impl Handler for Keyshare { // best practice would be as you boot up a node you enter in a configured password from // which we derive a kdf which gets used to generate this key self.data - .do_send(Insert(format!("{}/sk", e3_id).into(), sk)); + .write(Insert(format!("{}/sk", e3_id).into(), sk)); // save public key against e3_id/pk self.data - .do_send(Insert(format!("{}/pk", e3_id).into(), pubkey.clone())); + .write(Insert(format!("{}/pk", e3_id).into(), pubkey.clone())); // broadcast the KeyshareCreated message let event = EnclaveEvent::from(KeyshareCreated { @@ -109,7 +109,7 @@ impl Handler for Keyshare { async fn on_decryption_requested( fhe: Arc, - data: Addr, + data: DataStore, bus: Addr, event: CiphertextOutputPublished, address: String, @@ -120,7 +120,7 @@ async fn on_decryption_requested( } = event; // get secret key by id from data - let Some(unsafe_secret) = data.send(Get(format!("{}/sk", e3_id).into())).await? else { + let Some(unsafe_secret) = data.read(Get(format!("{}/sk", e3_id).into())).await? else { return Err(anyhow::anyhow!("Secret key not stored for {}", e3_id)); }; diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index 9e50c45a..f7847ee0 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -1,7 +1,7 @@ use crate::EventHook; use actix::{Actor, Addr}; use aggregator::{PlaintextAggregator, PublicKeyAggregator}; -use data::Data; +use data::DataStore; use enclave_core::{E3Requested, EnclaveEvent, EventBus}; use fhe::{Fhe, SharedRng}; use keyshare::Keyshare; @@ -28,7 +28,7 @@ impl LazyFhe { pub struct LazyKeyshare; impl LazyKeyshare { - pub fn create(bus: Addr, data: Addr, address: &str) -> EventHook { + pub fn create(bus: Addr, data: DataStore, address: &str) -> EventHook { let address = address.to_string(); Box::new(move |ctx, evt| { // Save Ciphernode on CiphernodeSelected diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index a6caa44c..88c3eafc 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -1,4 +1,4 @@ -use data::Data; +use data::{DataStore, InMemDataStore}; use enclave_core::{ CiphernodeAdded, CiphernodeSelected, CiphertextOutputPublished, DecryptionshareCreated, E3RequestComplete, E3Requested, E3id, EnclaveEvent, EventBus, GetHistory, KeyshareCreated, @@ -31,7 +31,8 @@ use tokio::{sync::mpsc::channel, time::sleep}; // Simulating a local node async fn setup_local_ciphernode(bus: Addr, rng: SharedRng, logging: bool, addr: &str) { // create data actor for saving data - let data = Data::new(logging).start(); // TODO: Use a sled backed Data Actor + let data_actor = InMemDataStore::new(logging).start(); // TODO: Use a sled backed Data Actor + let store = DataStore::from_in_mem(data_actor); // create ciphernode actor for managing ciphernode flow let sortition = Sortition::attach(bus.clone()); @@ -47,7 +48,7 @@ async fn setup_local_ciphernode(bus: Addr, rng: SharedRng, logging: bo bus.clone(), sortition.clone(), )) - .add_hook(LazyKeyshare::create(bus.clone(), data.clone(), addr)) + .add_hook(LazyKeyshare::create(bus.clone(), store.clone(), addr)) .build(); SimpleLogger::attach(addr, bus.clone()); From 164c98356379e1ee0857a893ad4441bc9ae93a44 Mon Sep 17 00:00:00 2001 From: ryardley Date: Mon, 14 Oct 2024 18:03:41 +1100 Subject: [PATCH 02/49] Add IntoKey for Datastore --- packages/ciphernode/data/src/data_store.rs | 112 ++++++++++++++++++- packages/ciphernode/data/src/lib.rs | 4 +- packages/ciphernode/keyshare/src/keyshare.rs | 3 +- 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index 90b4a5e1..fa813f35 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -3,10 +3,61 @@ use anyhow::Result; use crate::InMemDataStore; +pub trait IntoKey { + fn into_key(self) -> Vec; +} + +impl IntoKey for Vec { + fn into_key(self) -> Vec { + self.join("/").into_bytes() + } +} + +impl<'a> IntoKey for Vec<&'a str> { + fn into_key(self) -> Vec { + self.join("/").into_bytes() + } +} + +impl IntoKey for String { + fn into_key(self) -> Vec { + self.into_bytes() + } +} + +impl<'a> IntoKey for &'a str { + fn into_key(self) -> Vec { + self.as_bytes().to_vec() + } +} + +pub trait WithPrefix: Sized { + fn prefix(self, prefix: &str) -> Self; + fn base(self, key: &str) -> Self; +} + +impl WithPrefix for Vec { + fn prefix(self, prefix: &str) -> Self { + let Ok(encoded) = String::from_utf8(self.clone()) else { + // If this is not encoded as utf8 do nothing + return self; + }; + vec![prefix.to_string(), encoded].join("/").into_bytes() + } + + fn base(self, key: &str) -> Self { + key.to_string().into_bytes() + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "()")] pub struct Insert(pub Vec, pub Vec); impl Insert { + pub fn new(key: K, value: Vec) -> Self { + Self(key.into_key(), value) + } + pub fn key(&self) -> Vec { self.0.clone() } @@ -16,29 +67,61 @@ impl Insert { } } +impl WithPrefix for Insert { + fn prefix(self, prefix: &str) -> Self { + Insert(self.0.prefix(prefix), self.1) + } + + fn base(self, key: &str) -> Self { + Insert(self.0.base(key), self.1) + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "Option>")] pub struct Get(pub Vec); impl Get { + pub fn new(key: K) -> Self { + Self(key.into_key()) + } + pub fn key(&self) -> Vec { self.0.clone() } } +impl WithPrefix for Get { + fn prefix(self, prefix: &str) -> Self { + Get(self.0.prefix(prefix)) + } + fn base(self, key: &str) -> Self { + Get(self.0.base(key)) + } +} + #[derive(Clone)] -pub struct DataStore(Recipient, Recipient); +pub struct DataStore { + prefix: Option, + get: Recipient, + insert: Recipient, +} + impl DataStore { pub async fn read(&self, msg: Get) -> Result>> { - Ok(self.0.send(msg).await?) + Ok(self.get.send(msg).await?) } pub fn write(&self, msg: Insert) { - self.1.do_send(msg) + self.insert.do_send(msg) } // use this for testing pub fn from_in_mem(addr: Addr) -> Self { - Self(addr.clone().recipient(), addr.clone().recipient()) + Self { + get: addr.clone().recipient(), + insert: addr.clone().recipient(), + prefix: None, + } } // // use this for production @@ -47,3 +130,24 @@ impl DataStore { // Self(d.recipient(),d.recipient()) // } } + +impl WithPrefix for DataStore { + fn prefix(self, prefix: &str) -> Self { + Self { + get: self.get, + insert: self.insert, + prefix: self.prefix.map_or_else( + || Some(prefix.to_string()), + |p| Some(vec![prefix.to_string(), p].join("/")), + ), + } + } + + fn base(self, key: &str) -> Self { + Self { + get: self.get, + insert: self.insert, + prefix: Some(key.to_string()), + } + } +} diff --git a/packages/ciphernode/data/src/lib.rs b/packages/ciphernode/data/src/lib.rs index 6acf7a73..a439fca7 100644 --- a/packages/ciphernode/data/src/lib.rs +++ b/packages/ciphernode/data/src/lib.rs @@ -1,4 +1,4 @@ -mod in_mem; mod data_store; -pub use in_mem::*; +mod in_mem; pub use data_store::*; +pub use in_mem::*; diff --git a/packages/ciphernode/keyshare/src/keyshare.rs b/packages/ciphernode/keyshare/src/keyshare.rs index 113b828f..49a447f6 100644 --- a/packages/ciphernode/keyshare/src/keyshare.rs +++ b/packages/ciphernode/keyshare/src/keyshare.rs @@ -63,8 +63,7 @@ impl Handler for Keyshare { // reencrypt secretkey locally with env var - this is so we don't have to serialize a secret // best practice would be as you boot up a node you enter in a configured password from // which we derive a kdf which gets used to generate this key - self.data - .write(Insert(format!("{}/sk", e3_id).into(), sk)); + self.data.write(Insert(format!("{}/sk", e3_id).into(), sk)); // save public key against e3_id/pk self.data From 209ade0c8fcdf191a5fe3bc96fd5f6f3a32aba55 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 15 Oct 2024 10:08:07 +1100 Subject: [PATCH 03/49] Tidy up writer API --- packages/ciphernode/Cargo.lock | 3 + packages/ciphernode/data/src/data_store.rs | 23 +++- .../ciphernode/enclave_node/src/aggregator.rs | 4 +- .../ciphernode/enclave_node/src/ciphernode.rs | 3 +- packages/ciphernode/fhe/src/fhe.rs | 29 ++++- packages/ciphernode/keyshare/src/keyshare.rs | 6 +- packages/ciphernode/router/Cargo.toml | 3 + .../ciphernode/router/src/committee_meta.rs | 16 ++- .../router/src/e3_request_router.rs | 116 ++++++++++++++++-- packages/ciphernode/router/src/hooks.rs | 65 +++++++--- .../tests/test_aggregation_and_decryption.rs | 4 +- 11 files changed, 222 insertions(+), 50 deletions(-) diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 8738f222..7a9d5d2c 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -5120,10 +5120,13 @@ version = "0.1.0" dependencies = [ "actix", "aggregator", + "anyhow", + "bincode", "data", "enclave-core", "fhe 0.1.0", "keyshare", + "serde", "sortition", ] diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index fa813f35..6a33c974 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -1,5 +1,5 @@ use actix::{Addr, Message, Recipient}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use crate::InMemDataStore; @@ -25,6 +25,12 @@ impl IntoKey for String { } } +impl IntoKey for &String { + fn into_key(self) -> Vec { + self.as_bytes().to_vec() + } +} + impl<'a> IntoKey for &'a str { fn into_key(self) -> Vec { self.as_bytes().to_vec() @@ -107,11 +113,15 @@ pub struct DataStore { } impl DataStore { - pub async fn read(&self, msg: Get) -> Result>> { + pub async fn read(&self, key: K) -> Result>> { + let msg = Get::new(key); + let msg = self.prefix.as_ref().map_or(msg.clone(), |p| msg.prefix(p)); Ok(self.get.send(msg).await?) } - pub fn write(&self, msg: Insert) { + pub fn write(&self, key: K, value: Vec) { + let msg = Insert::new(key, value); + let msg = self.prefix.as_ref().map_or(msg.clone(), |p| msg.prefix(p)); self.insert.do_send(msg) } @@ -124,6 +134,13 @@ impl DataStore { } } + pub fn ensure_root_id(str: &str) -> Result<()> { + if !str.starts_with("/") { + return Err(anyhow!("string doesnt start with slash.")); + } + Ok(()) + } + // // use this for production // pub fn from_sled(&data_addr: Addr) -> Self { // let d = data_addr.clone(); diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 965e467f..78ea1d3d 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -1,5 +1,6 @@ use actix::{Actor, Addr, Context}; use anyhow::Result; +use data::{DataStore, InMemDataStore}; use enclave_core::EventBus; use evm::{ helpers::pull_eth_signer_from_env, CiphernodeRegistrySol, EnclaveSol, RegistryFilterSol, @@ -51,6 +52,7 @@ impl MainAggregator { rand_chacha::ChaCha20Rng::from_rng(OsRng).expect("Failed to create RNG"), )); + let store = DataStore::from_in_mem(InMemDataStore::new(true).start()); let sortition = Sortition::attach(bus.clone()); let signer = pull_eth_signer_from_env("PRIVATE_KEY").await?; for chain in config @@ -81,7 +83,7 @@ impl MainAggregator { .await?; } - let e3_manager = E3RequestRouter::builder(bus.clone()) + let e3_manager = E3RequestRouter::builder(bus.clone(), store) .add_hook(LazyFhe::create(rng)) .add_hook(LazyPublicKeyAggregator::create( bus.clone(), diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 43f59e2c..09a0af26 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -79,11 +79,10 @@ impl MainCiphernode { .await?; } - let e3_manager = E3RequestRouter::builder(bus.clone()) + let e3_manager = E3RequestRouter::builder(bus.clone(), data.clone()) .add_hook(LazyFhe::create(rng)) .add_hook(LazyKeyshare::create( bus.clone(), - data.clone(), &address.to_string(), )) .build(); diff --git a/packages/ciphernode/fhe/src/fhe.rs b/packages/ciphernode/fhe/src/fhe.rs index 3cab9650..fde895aa 100644 --- a/packages/ciphernode/fhe/src/fhe.rs +++ b/packages/ciphernode/fhe/src/fhe.rs @@ -1,6 +1,6 @@ use super::set_up_crp; use anyhow::*; -use enclave_core::{E3Requested, EnclaveEvent, OrderedSet, Seed}; +use enclave_core::{ OrderedSet, Seed}; use fhe_rs::{ bfv::{ BfvParameters, BfvParametersBuilder, Ciphertext, Encoding, Plaintext, PublicKey, SecretKey, @@ -31,8 +31,8 @@ pub type SharedRng = Arc>; /// Fhe library adaptor. #[derive(Clone)] pub struct Fhe { - params: Arc, - crp: CommonRandomPoly, + pub params: Arc, + pub crp: CommonRandomPoly, rng: SharedRng, } @@ -47,7 +47,21 @@ impl Fhe { params.clone(), Arc::new(Mutex::new(ChaCha20Rng::from_seed(seed.into()))), ); - Ok(Fhe::new(params.clone(), crp, rng.clone())) + Ok(Fhe::new(params, crp, rng)) + } + + pub fn to_bytes(&self) -> Result> { + Ok(bincode::serialize(&PackedParams { + crp: self.crp.to_bytes(), + params: self.params.to_bytes(), + })?) + } + + pub fn from_bytes(packed: &[u8], rng:SharedRng) -> Result { + let unpacked: PackedParams = bincode::deserialize(packed)?; + let params = Arc::new(BfvParameters::try_deserialize(&unpacked.params)?); + let crp = CommonRandomPoly::deserialize(&unpacked.crp, ¶ms)?; + Ok(Fhe::new(params, crp, rng)) } pub fn from_raw_params( @@ -122,6 +136,13 @@ impl Fhe { Ok(bincode::serialize(&decoded)?) } } + +#[derive(serde::Serialize, serde::Deserialize)] +struct PackedParams { + crp: Vec, + params: Vec, +} + struct SecretKeySerializer { pub inner: SecretKey, } diff --git a/packages/ciphernode/keyshare/src/keyshare.rs b/packages/ciphernode/keyshare/src/keyshare.rs index 49a447f6..98e0cb94 100644 --- a/packages/ciphernode/keyshare/src/keyshare.rs +++ b/packages/ciphernode/keyshare/src/keyshare.rs @@ -63,11 +63,11 @@ impl Handler for Keyshare { // reencrypt secretkey locally with env var - this is so we don't have to serialize a secret // best practice would be as you boot up a node you enter in a configured password from // which we derive a kdf which gets used to generate this key - self.data.write(Insert(format!("{}/sk", e3_id).into(), sk)); + self.data.write(format!("{e3_id}/sk"), sk); // save public key against e3_id/pk self.data - .write(Insert(format!("{}/pk", e3_id).into(), pubkey.clone())); + .write(format!("{e3_id}/pk"), pubkey.clone()); // broadcast the KeyshareCreated message let event = EnclaveEvent::from(KeyshareCreated { @@ -119,7 +119,7 @@ async fn on_decryption_requested( } = event; // get secret key by id from data - let Some(unsafe_secret) = data.read(Get(format!("{}/sk", e3_id).into())).await? else { + let Some(unsafe_secret) = data.read(&format!("{e3_id}/sk")).await? else { return Err(anyhow::anyhow!("Secret key not stored for {}", e3_id)); }; diff --git a/packages/ciphernode/router/Cargo.toml b/packages/ciphernode/router/Cargo.toml index aef11dc7..286095d8 100644 --- a/packages/ciphernode/router/Cargo.toml +++ b/packages/ciphernode/router/Cargo.toml @@ -11,3 +11,6 @@ fhe = { path = "../fhe" } data = { path = "../data" } keyshare = { path = "../keyshare" } aggregator = { path = "../aggregator" } +anyhow = { workspace = true } +serde = { workspace = true } +bincode = { workspace = true } diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index 79d3c314..c90b5f15 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -2,7 +2,7 @@ use enclave_core::{E3Requested, EnclaveEvent, Seed}; use super::EventHook; -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct CommitteeMeta { pub threshold_m: usize, pub seed: Seed, @@ -21,14 +21,18 @@ impl CommitteeMetaFactory { threshold_m, seed, src_chain_id, + e3_id, .. } = data; - ctx.meta = Some(CommitteeMeta { - threshold_m, - seed, - src_chain_id, - }); + ctx.set_meta( + &format!("//meta/{e3_id}"), + CommitteeMeta { + threshold_m, + seed, + src_chain_id, + }, + ); }) } } diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index dce2b395..92441a52 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -1,9 +1,13 @@ -use crate::CommitteeMetaFactory; - use super::CommitteeMeta; +use crate::CommitteeMetaFactory; use actix::{Actor, Addr, Context, Handler, Recipient}; use aggregator::PlaintextAggregator; use aggregator::PublicKeyAggregator; +use anyhow::*; +use data::DataStore; +use data::Get; +use data::Insert; +use data::WithPrefix; use enclave_core::E3RequestComplete; use enclave_core::{E3id, EnclaveEvent, EventBus, Subscribe}; use fhe::Fhe; @@ -32,16 +36,27 @@ impl EventBuffer { /// Context that is set to each event hook. Hooks can use this context to gather dependencies if /// they need to instantiate struct instances or actors. -#[derive(Default)] pub struct E3RequestContext { - pub keyshare: Option>, - pub fhe: Option>, - pub plaintext: Option>, - pub publickey: Option>, - pub meta: Option, + keyshare: Option>, + fhe: Option>, + plaintext: Option>, + publickey: Option>, + meta: Option, + pub store: DataStore, } impl E3RequestContext { + fn from_store(store: DataStore) -> Self { + Self { + keyshare: None, + fhe: None, + plaintext: None, + publickey: None, + meta: None, + store, + } + } + fn recipients(&self) -> Vec<(String, Option>)> { vec![ ( @@ -71,6 +86,71 @@ impl E3RequestContext { } }); } + + /// Accept a DataStore ID and a Keystore actor address + pub fn set_keyshare(&mut self, id: &str, value: Addr) -> Result<()> { + DataStore::ensure_root_id(id)?; + self.keyshare = Some(value); + self.store.write("keyshare", id.into()); + Ok(()) + } + + /// Accept a DataStore ID and a Keystore actor address + pub fn set_plaintext(&mut self, id: &str, value: Addr) -> Result<()> { + DataStore::ensure_root_id(id)?; + self.plaintext = Some(value); + self.store.write("plaintext", id.into()); + Ok(()) + } + + /// Accept a DataStore ID and a Keystore actor address + pub fn set_publickey(&mut self, id: &str, value: Addr) -> Result<()> { + DataStore::ensure_root_id(id)?; + self.publickey = Some(value); + self.store.write("publickey", id.into()); + Ok(()) + } + + /// Accept a DataStore ID and an Arc instance of the Fhe wrapper + pub fn set_fhe(&mut self, id: &str, value: Arc) -> Result<()> { + DataStore::ensure_root_id(id)?; + self.fhe = Some(value.clone()); + self.store.write("fhe", id.into()); + // TODO: should non actors store themselves? + self.store.write(id, value.to_bytes()?); + Ok(()) + } + + /// Accept a Datastore ID and a metadata object + pub fn set_meta(&mut self, id: &str, value: CommitteeMeta) -> Result<()> { + DataStore::ensure_root_id(id)?; + self.meta = Some(value.clone()); + self.store.write("meta", id.into()); + // TODO: should non actors store themselves? + self.store + .write(id, bincode::serialize(&value)?); + Ok(()) + } + + pub fn get_keyshare(&self) -> Option<&Addr> { + self.keyshare.as_ref() + } + + pub fn get_plaintext(&self) -> Option<&Addr> { + self.plaintext.as_ref() + } + + pub fn get_publickey(&self) -> Option<&Addr> { + self.publickey.as_ref() + } + + pub fn get_fhe(&self) -> Option<&Arc> { + self.fhe.as_ref() + } + + pub fn get_meta(&self) -> Option<&CommitteeMeta> { + self.meta.as_ref() + } } /// Format of the hook that needs to be passed to E3RequestRouter @@ -88,11 +168,16 @@ pub struct E3RequestRouter { hooks: Vec, buffer: EventBuffer, bus: Addr, + store: DataStore, } impl E3RequestRouter { - pub fn builder(bus: Addr) -> E3RequestRouterBuilder { - let builder = E3RequestRouterBuilder { bus, hooks: vec![] }; + pub fn builder(bus: Addr, store: DataStore) -> E3RequestRouterBuilder { + let builder = E3RequestRouterBuilder { + bus, + hooks: vec![], + store, + }; // Everything needs the committe meta factory so adding it here by default builder.add_hook(CommitteeMetaFactory::create()) @@ -115,7 +200,10 @@ impl Handler for E3RequestRouter { return; } - let context = self.contexts.entry(e3_id.clone()).or_default(); + let context = self + .contexts + .entry(e3_id.clone()) + .or_insert_with(|| E3RequestContext::from_store(self.store.clone())); for hook in &mut self.hooks { hook(context, msg.clone()); @@ -150,6 +238,7 @@ impl Handler for E3RequestRouter { pub struct E3RequestRouterBuilder { pub bus: Addr, pub hooks: Vec, + pub store: DataStore, } impl E3RequestRouterBuilder { pub fn add_hook(mut self, listener: EventHook) -> Self { @@ -164,6 +253,7 @@ impl E3RequestRouterBuilder { hooks: self.hooks, buffer: EventBuffer::default(), bus: self.bus.clone(), + store: self.store, }; let addr = e3r.start(); @@ -171,4 +261,8 @@ impl E3RequestRouterBuilder { .do_send(Subscribe::new("*", addr.clone().recipient())); addr } + + // pub async fn hydrate(self) -> Addr { + // let store = self.store.base("//router"); + // } } diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index f7847ee0..9c2345b1 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -1,7 +1,7 @@ use crate::EventHook; use actix::{Actor, Addr}; use aggregator::{PlaintextAggregator, PublicKeyAggregator}; -use data::DataStore; +use data::{DataStore, WithPrefix}; use enclave_core::{E3Requested, EnclaveEvent, EventBus}; use fhe::{Fhe, SharedRng}; use keyshare::Keyshare; @@ -17,31 +17,51 @@ impl LazyFhe { let EnclaveEvent::E3Requested { data, .. } = evt else { return; }; - let E3Requested { params, seed, .. } = data; - ctx.fhe = Some(Arc::new( - Fhe::from_encoded(¶ms, seed, rng.clone()).unwrap(), - )); + let E3Requested { + params, + seed, + e3_id, + .. + } = data; + + // Set the FHE instance passing in the instance id + let _ = ctx.set_fhe( + &format!("//fhe/{e3_id}"), + Arc::new(Fhe::from_encoded(¶ms, seed, rng.clone()).unwrap()), + ); }) } } pub struct LazyKeyshare; impl LazyKeyshare { - pub fn create(bus: Addr, data: DataStore, address: &str) -> EventHook { + pub fn create(bus: Addr, address: &str) -> EventHook { let address = address.to_string(); Box::new(move |ctx, evt| { // Save Ciphernode on CiphernodeSelected - let EnclaveEvent::CiphernodeSelected { .. } = evt else { + let EnclaveEvent::CiphernodeSelected { data, .. } = evt else { return; }; - let Some(ref fhe) = ctx.fhe else { + let Some(fhe) = ctx.get_fhe() else { return; }; - ctx.keyshare = - Some(Keyshare::new(bus.clone(), data.clone(), fhe.clone(), &address).start()) + let e3_id = data.e3_id; + + let ks_id = &format!("//keystore/{e3_id}"); + + let _ = ctx.set_keyshare( + ks_id, + Keyshare::new( + bus.clone(), + ctx.store.clone().base(ks_id), + fhe.clone(), + &address, + ) + .start(), + ); }) } } @@ -54,19 +74,24 @@ impl LazyPlaintextAggregator { let EnclaveEvent::CiphertextOutputPublished { data, .. } = evt else { return; }; - let Some(ref fhe) = ctx.fhe else { + let Some(fhe) = ctx.get_fhe() else { return; }; - let Some(ref meta) = ctx.meta else { + let Some(ref meta) = ctx.get_meta() else { return; }; - ctx.plaintext = Some( + let e3_id = data.e3_id; + + let id = &format!("//plaintext/{e3_id}"); + + let _ = ctx.set_plaintext( + id, PlaintextAggregator::new( fhe.clone(), bus.clone(), sortition.clone(), - data.e3_id, + e3_id, meta.threshold_m, meta.seed, data.ciphertext_output, @@ -87,21 +112,25 @@ impl LazyPublicKeyAggregator { return; }; - let Some(ref fhe) = ctx.fhe else { + let Some(fhe) = ctx.get_fhe() else { println!("fhe was not on ctx"); return; }; - let Some(ref meta) = ctx.meta else { + let Some(ref meta) = ctx.get_meta() else { println!("meta was not on ctx"); return; }; - ctx.publickey = Some( + let e3_id = data.e3_id; + let id = &format!("//publickey/{e3_id}"); + + let _ = ctx.set_publickey( + id, PublicKeyAggregator::new( fhe.clone(), bus.clone(), sortition.clone(), - data.e3_id, + e3_id, meta.threshold_m, meta.seed, meta.src_chain_id, diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 88c3eafc..4232a2ae 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -38,7 +38,7 @@ async fn setup_local_ciphernode(bus: Addr, rng: SharedRng, logging: bo let sortition = Sortition::attach(bus.clone()); CiphernodeSelector::attach(bus.clone(), sortition.clone(), addr); - E3RequestRouter::builder(bus.clone()) + E3RequestRouter::builder(bus.clone(), store) .add_hook(LazyFhe::create(rng)) .add_hook(LazyPublicKeyAggregator::create( bus.clone(), @@ -48,7 +48,7 @@ async fn setup_local_ciphernode(bus: Addr, rng: SharedRng, logging: bo bus.clone(), sortition.clone(), )) - .add_hook(LazyKeyshare::create(bus.clone(), store.clone(), addr)) + .add_hook(LazyKeyshare::create(bus.clone(), addr)) .build(); SimpleLogger::attach(addr, bus.clone()); From 2abe37297b55f78e131318ec6d36aa476afe89b8 Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 15 Oct 2024 19:17:15 +1100 Subject: [PATCH 04/49] Apply Checkpoint API --- packages/ciphernode/Cargo.lock | 11 ++ packages/ciphernode/aggregator/Cargo.toml | 3 + .../aggregator/src/plaintext_aggregator.rs | 83 ++++++--- .../aggregator/src/publickey_aggregator.rs | 77 +++++--- packages/ciphernode/data/Cargo.toml | 3 + packages/ciphernode/data/src/data_store.rs | 106 ++++++++--- packages/ciphernode/fhe/Cargo.toml | 8 +- packages/ciphernode/fhe/src/fhe.rs | 40 +++-- packages/ciphernode/keyshare/Cargo.toml | 2 + packages/ciphernode/keyshare/src/keyshare.rs | 169 ++++++++++-------- packages/ciphernode/router/Cargo.toml | 1 + .../ciphernode/router/src/committee_meta.rs | 19 +- .../router/src/e3_request_router.rs | 131 ++++++++++---- packages/ciphernode/router/src/hooks.rs | 74 ++++---- 14 files changed, 494 insertions(+), 233 deletions(-) diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 7a9d5d2c..52331a95 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -115,9 +115,12 @@ version = "0.1.0" dependencies = [ "actix", "anyhow", + "async-trait", "bincode", + "data", "enclave-core", "fhe 0.1.0", + "serde", "sortition", ] @@ -1842,6 +1845,9 @@ version = "0.1.0" dependencies = [ "actix", "anyhow", + "async-trait", + "bincode", + "serde", ] [[package]] @@ -2284,7 +2290,9 @@ name = "fhe" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "bincode", + "data", "enclave-core", "fhe 0.1.0-beta.7", "fhe-traits", @@ -3176,9 +3184,11 @@ version = "0.1.0" dependencies = [ "actix", "anyhow", + "async-trait", "data", "enclave-core", "fhe 0.1.0", + "serde", ] [[package]] @@ -5121,6 +5131,7 @@ dependencies = [ "actix", "aggregator", "anyhow", + "async-trait", "bincode", "data", "enclave-core", diff --git a/packages/ciphernode/aggregator/Cargo.toml b/packages/ciphernode/aggregator/Cargo.toml index 695e6bf6..fe3f76bc 100644 --- a/packages/ciphernode/aggregator/Cargo.toml +++ b/packages/ciphernode/aggregator/Cargo.toml @@ -6,7 +6,10 @@ edition = "2021" [dependencies] actix = { workspace = true } anyhow = { workspace = true } +serde = { workspace = true } bincode = { workspace = true } +async-trait = { workspace = true } enclave-core = { path = "../core" } fhe = { path = "../fhe" } sortition = { path = "../sortition" } +data = { path = "../data" } diff --git a/packages/ciphernode/aggregator/src/plaintext_aggregator.rs b/packages/ciphernode/aggregator/src/plaintext_aggregator.rs index c63a4dda..50d3f6c5 100644 --- a/packages/ciphernode/aggregator/src/plaintext_aggregator.rs +++ b/packages/ciphernode/aggregator/src/plaintext_aggregator.rs @@ -1,14 +1,16 @@ use actix::prelude::*; use anyhow::Result; +use async_trait::async_trait; +use data::{Checkpoint, DataStore, FromSnapshotWithParams, Snapshot}; use enclave_core::{ - DecryptionshareCreated, Die, E3RequestComplete, E3id, EnclaveEvent, EventBus, OrderedSet, - PlaintextAggregated, Seed, + DecryptionshareCreated, Die, E3id, EnclaveEvent, EventBus, OrderedSet, PlaintextAggregated, + Seed, }; use fhe::{Fhe, GetAggregatePlaintext}; use sortition::{GetHasNode, Sortition}; use std::sync::Arc; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum PlaintextAggregatorState { Collecting { threshold_m: usize, @@ -26,6 +28,17 @@ pub enum PlaintextAggregatorState { }, } +impl PlaintextAggregatorState { + pub fn init(threshold_m: usize, seed: Seed, ciphertext_output: Vec) -> Self { + PlaintextAggregatorState::Collecting { + threshold_m, + shares: OrderedSet::new(), + seed, + ciphertext_output, + } + } +} + #[derive(Message)] #[rtype(result = "anyhow::Result<()>")] struct ComputeAggregate { @@ -36,35 +49,32 @@ struct ComputeAggregate { pub struct PlaintextAggregator { fhe: Arc, bus: Addr, + store: DataStore, sortition: Addr, e3_id: E3id, state: PlaintextAggregatorState, src_chain_id: u64, } +pub struct PlaintextAggregatorParams { + pub fhe: Arc, + pub bus: Addr, + pub store: DataStore, + pub sortition: Addr, + pub e3_id: E3id, + pub src_chain_id: u64, +} + impl PlaintextAggregator { - pub fn new( - fhe: Arc, - bus: Addr, - sortition: Addr, - e3_id: E3id, - threshold_m: usize, - seed: Seed, - ciphertext_output: Vec, - src_chain_id: u64, - ) -> Self { + pub fn new(params: PlaintextAggregatorParams, state: PlaintextAggregatorState) -> Self { PlaintextAggregator { - fhe, - bus, - sortition, - e3_id, - src_chain_id, - state: PlaintextAggregatorState::Collecting { - threshold_m, - shares: OrderedSet::new(), - seed, - ciphertext_output, - }, + fhe: params.fhe, + bus: params.bus, + store: params.store, + sortition: params.sortition, + e3_id: params.e3_id, + src_chain_id: params.src_chain_id, + state, } } @@ -155,6 +165,7 @@ impl Handler for PlaintextAggregator { // add the keyshare and act.state = act.add_share(decryption_share)?; + act.checkpoint(); // Check the state and if it has changed to the computing if let PlaintextAggregatorState::Computing { @@ -184,6 +195,7 @@ impl Handler for PlaintextAggregator { // Update the local state self.state = self.set_decryption(decrypted_output.clone())?; + self.checkpoint(); // Dispatch the PlaintextAggregated event let event = EnclaveEvent::from(PlaintextAggregated { @@ -204,3 +216,26 @@ impl Handler for PlaintextAggregator { ctx.stop() } } + +impl Snapshot for PlaintextAggregator { + type Snapshot = PlaintextAggregatorState; + + fn snapshot(&self) -> Self::Snapshot { + self.state.clone() + } +} + +#[async_trait] +impl FromSnapshotWithParams for PlaintextAggregator { + type Params = PlaintextAggregatorParams; + + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + Ok(PlaintextAggregator::new(params, snapshot)) + } +} + +impl Checkpoint for PlaintextAggregator { + fn get_store(&self) -> data::DataStore { + self.store.clone() + } +} diff --git a/packages/ciphernode/aggregator/src/publickey_aggregator.rs b/packages/ciphernode/aggregator/src/publickey_aggregator.rs index 7274dafe..deb05366 100644 --- a/packages/ciphernode/aggregator/src/publickey_aggregator.rs +++ b/packages/ciphernode/aggregator/src/publickey_aggregator.rs @@ -1,5 +1,7 @@ use actix::prelude::*; use anyhow::Result; +use async_trait::async_trait; +use data::{Checkpoint, DataStore, FromSnapshotWithParams, Snapshot}; use enclave_core::{ Die, E3id, EnclaveEvent, EventBus, KeyshareCreated, OrderedSet, PublicKeyAggregated, Seed, }; @@ -7,7 +9,7 @@ use fhe::{Fhe, GetAggregatePublicKey}; use sortition::{GetHasNode, GetNodes, Sortition}; use std::sync::Arc; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum PublicKeyAggregatorState { Collecting { threshold_m: usize, @@ -23,6 +25,16 @@ pub enum PublicKeyAggregatorState { }, } +impl PublicKeyAggregatorState { + pub fn init(threshold_m: usize, seed: Seed) -> Self { + PublicKeyAggregatorState::Collecting { + threshold_m, + keyshares: OrderedSet::new(), + seed, + } + } +} + #[derive(Message)] #[rtype(result = "anyhow::Result<()>")] struct ComputeAggregate { @@ -40,12 +52,22 @@ struct NotifyNetwork { pub struct PublicKeyAggregator { fhe: Arc, bus: Addr, + store: DataStore, sortition: Addr, e3_id: E3id, state: PublicKeyAggregatorState, src_chain_id: u64, } +pub struct PublicKeyAggregatorParams { + pub fhe: Arc, + pub bus: Addr, + pub store: DataStore, + pub sortition: Addr, + pub e3_id: E3id, + pub src_chain_id: u64, +} + /// Aggregate PublicKey for a committee of nodes. This actor listens for KeyshareCreated events /// around a particular e3_id and aggregates the public key based on this and once done broadcasts /// a EnclaveEvent::PublicKeyAggregated event on the event bus. Note events are hashed and @@ -53,26 +75,15 @@ pub struct PublicKeyAggregator { /// It is expected to change this mechanism as we work through adversarial scenarios and write tests /// for them. impl PublicKeyAggregator { - pub fn new( - fhe: Arc, - bus: Addr, - sortition: Addr, - e3_id: E3id, - threshold_m: usize, - seed: Seed, - src_chain_id: u64, - ) -> Self { + pub fn new(params: PublicKeyAggregatorParams, state: PublicKeyAggregatorState) -> Self { PublicKeyAggregator { - fhe, - bus, - e3_id, - sortition, - src_chain_id, - state: PublicKeyAggregatorState::Collecting { - threshold_m, - keyshares: OrderedSet::new(), - seed, - }, + fhe: params.fhe, + bus: params.bus, + store: params.store, + sortition: params.sortition, + e3_id: params.e3_id, + src_chain_id: params.src_chain_id, + state, } } @@ -166,6 +177,7 @@ impl Handler for PublicKeyAggregator { // add the keyshare and act.state = act.add_keyshare(pubkey)?; + act.checkpoint(); // Check the state and if it has changed to the computing if let PublicKeyAggregatorState::Computing { keyshares } = &act.state { @@ -191,6 +203,8 @@ impl Handler for PublicKeyAggregator { // Update the local state self.state = self.set_pubkey(pubkey.clone())?; + self.checkpoint(); + ctx.notify(NotifyNetwork { pubkey, e3_id: msg.e3_id, @@ -228,3 +242,26 @@ impl Handler for PublicKeyAggregator { ctx.stop() } } + +impl Snapshot for PublicKeyAggregator { + type Snapshot = PublicKeyAggregatorState; + + fn snapshot(&self) -> Self::Snapshot { + self.state.clone() + } +} + +#[async_trait] +impl FromSnapshotWithParams for PublicKeyAggregator { + type Params = PublicKeyAggregatorParams; + + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + Ok(PublicKeyAggregator::new(params, snapshot)) + } +} + +impl Checkpoint for PublicKeyAggregator { + fn get_store(&self) -> data::DataStore { + self.store.clone() + } +} diff --git a/packages/ciphernode/data/Cargo.toml b/packages/ciphernode/data/Cargo.toml index a69223fd..dbac17ad 100644 --- a/packages/ciphernode/data/Cargo.toml +++ b/packages/ciphernode/data/Cargo.toml @@ -8,3 +8,6 @@ repository = "https://github.com/gnosisguild/enclave/packages/ciphernode" [dependencies] actix = { workspace = true } anyhow = { workspace = true } +serde = { workspace = true } +bincode = { workspace = true } +async-trait = { workspace = true } diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index 6a33c974..aa58aabc 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -1,7 +1,8 @@ +use crate::InMemDataStore; use actix::{Addr, Message, Recipient}; use anyhow::{anyhow, Result}; - -use crate::InMemDataStore; +use async_trait::async_trait; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; pub trait IntoKey { fn into_key(self) -> Vec; @@ -38,20 +39,57 @@ impl<'a> IntoKey for &'a str { } pub trait WithPrefix: Sized { - fn prefix(self, prefix: &str) -> Self; - fn base(self, key: &str) -> Self; + fn prefix(&self, prefix: &str) -> Self; + fn at(&self, key: &str) -> Self; +} + +pub trait Snapshot +where + Self: Sized, +{ + type Snapshot: Serialize + DeserializeOwned; + + /// Return a tuple with the first element being the id string of the object and the second + /// being a representation of the object's state that is easily serialized by the data store + fn snapshot(&self) -> Self::Snapshot; +} + +pub trait Checkpoint: Snapshot { + /// Declare the DataStore instance available on the object + fn get_store(&self) -> DataStore; + + /// Write the current snapshot to the DataStore provided by `get_store()` at the object's id returned by `get_id()` + fn checkpoint(&self) { + self.get_store().write(self.snapshot()); + } +} + +#[async_trait] +pub trait FromSnapshotWithParams: Snapshot { + type Params; + + /// Return an instance of the persistable object at the state given by the snapshot + /// This method is async because there may be subobjects that require hydration from the store + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result; +} + +#[async_trait] +pub trait FromSnapshot: Snapshot { + /// Return an instance of the persistable object at the state given by the snapshot + /// This method is async because there may be subobjects that require hydration from the store + async fn from_snapshot(snapshot: Self::Snapshot) -> Result; } impl WithPrefix for Vec { - fn prefix(self, prefix: &str) -> Self { + fn prefix(&self, prefix: &str) -> Self { let Ok(encoded) = String::from_utf8(self.clone()) else { // If this is not encoded as utf8 do nothing - return self; + return self.clone(); }; vec![prefix.to_string(), encoded].join("/").into_bytes() } - fn base(self, key: &str) -> Self { + fn at(&self, key: &str) -> Self { key.to_string().into_bytes() } } @@ -74,12 +112,12 @@ impl Insert { } impl WithPrefix for Insert { - fn prefix(self, prefix: &str) -> Self { - Insert(self.0.prefix(prefix), self.1) + fn prefix(&self, prefix: &str) -> Self { + Insert(self.0.prefix(prefix), self.1.clone()) } - fn base(self, key: &str) -> Self { - Insert(self.0.base(key), self.1) + fn at(&self, key: &str) -> Self { + Insert(self.0.at(key), self.1.clone()) } } @@ -97,11 +135,11 @@ impl Get { } impl WithPrefix for Get { - fn prefix(self, prefix: &str) -> Self { + fn prefix(&self, prefix: &str) -> Self { Get(self.0.prefix(prefix)) } - fn base(self, key: &str) -> Self { - Get(self.0.base(key)) + fn at(&self, key: &str) -> Self { + Get(self.0.at(key)) } } @@ -113,18 +151,36 @@ pub struct DataStore { } impl DataStore { - pub async fn read(&self, key: K) -> Result>> { + pub async fn read(&self, key: K) -> Result> + where + K: IntoKey, + T: for<'de> Deserialize<'de>, + { let msg = Get::new(key); let msg = self.prefix.as_ref().map_or(msg.clone(), |p| msg.prefix(p)); - Ok(self.get.send(msg).await?) + let maybe_bytes = self.get.send(msg).await?; + let Some(bytes) = maybe_bytes else { + return Ok(None); + }; + + Ok(Some(bincode::deserialize(&bytes)?)) } - pub fn write(&self, key: K, value: Vec) { - let msg = Insert::new(key, value); + /// Writes anything serializable to the KV actor as a stream of bytes + pub fn set(&self, key: K, value: V) { + let Ok(serialized) = bincode::serialize(&value) else { + return; + }; + let msg = Insert::new(key, serialized); let msg = self.prefix.as_ref().map_or(msg.clone(), |p| msg.prefix(p)); self.insert.do_send(msg) } + /// Writes to whatever the prefix is set to on the datastore + pub fn write(&self, value: V) { + self.set("", value) + } + // use this for testing pub fn from_in_mem(addr: Addr) -> Self { Self { @@ -149,21 +205,21 @@ impl DataStore { } impl WithPrefix for DataStore { - fn prefix(self, prefix: &str) -> Self { + fn prefix(&self, prefix: &str) -> Self { Self { - get: self.get, - insert: self.insert, - prefix: self.prefix.map_or_else( + get: self.get.clone(), + insert: self.insert.clone(), + prefix: self.prefix.clone().map_or_else( || Some(prefix.to_string()), |p| Some(vec![prefix.to_string(), p].join("/")), ), } } - fn base(self, key: &str) -> Self { + fn at(&self, key: &str) -> Self { Self { - get: self.get, - insert: self.insert, + get: self.get.clone(), + insert: self.insert.clone(), prefix: Some(key.to_string()), } } diff --git a/packages/ciphernode/fhe/Cargo.toml b/packages/ciphernode/fhe/Cargo.toml index 05cf7414..3272964a 100644 --- a/packages/ciphernode/fhe/Cargo.toml +++ b/packages/ciphernode/fhe/Cargo.toml @@ -4,12 +4,14 @@ version = "0.1.0" edition = "2021" [dependencies] -enclave-core = { path = "../core" } anyhow = { workspace = true } -fhe_rs = { workspace = true } +async-trait = { workspace = true } +bincode = { workspace = true } +data = { path = "../data" } +enclave-core = { path = "../core" } fhe-traits = { workspace = true } fhe-util = { workspace = true } +fhe_rs = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } -bincode = { workspace = true } serde = { workspace = true } diff --git a/packages/ciphernode/fhe/src/fhe.rs b/packages/ciphernode/fhe/src/fhe.rs index fde895aa..99fd8112 100644 --- a/packages/ciphernode/fhe/src/fhe.rs +++ b/packages/ciphernode/fhe/src/fhe.rs @@ -1,6 +1,8 @@ use super::set_up_crp; use anyhow::*; -use enclave_core::{ OrderedSet, Seed}; +use async_trait::async_trait; +use data::{FromSnapshotWithParams, Snapshot}; +use enclave_core::{OrderedSet, Seed}; use fhe_rs::{ bfv::{ BfvParameters, BfvParametersBuilder, Ciphertext, Encoding, Plaintext, PublicKey, SecretKey, @@ -50,20 +52,6 @@ impl Fhe { Ok(Fhe::new(params, crp, rng)) } - pub fn to_bytes(&self) -> Result> { - Ok(bincode::serialize(&PackedParams { - crp: self.crp.to_bytes(), - params: self.params.to_bytes(), - })?) - } - - pub fn from_bytes(packed: &[u8], rng:SharedRng) -> Result { - let unpacked: PackedParams = bincode::deserialize(packed)?; - let params = Arc::new(BfvParameters::try_deserialize(&unpacked.params)?); - let crp = CommonRandomPoly::deserialize(&unpacked.crp, ¶ms)?; - Ok(Fhe::new(params, crp, rng)) - } - pub fn from_raw_params( moduli: &[u64], degree: usize, @@ -137,8 +125,28 @@ impl Fhe { } } +impl Snapshot for Fhe { + type Snapshot = FheSnapshot; + fn snapshot(&self) -> Self::Snapshot { + FheSnapshot { + crp: self.crp.to_bytes(), + params: self.params.to_bytes(), + } + } +} + +#[async_trait] +impl FromSnapshotWithParams for Fhe { + type Params = SharedRng; + async fn from_snapshot(rng: SharedRng, snapshot: FheSnapshot) -> Result { + let params = Arc::new(BfvParameters::try_deserialize(&snapshot.params).unwrap()); + let crp = CommonRandomPoly::deserialize(&snapshot.crp, ¶ms)?; + Ok(Fhe::new(params, crp, rng)) + } +} + #[derive(serde::Serialize, serde::Deserialize)] -struct PackedParams { +pub struct FheSnapshot { crp: Vec, params: Vec, } diff --git a/packages/ciphernode/keyshare/Cargo.toml b/packages/ciphernode/keyshare/Cargo.toml index 29ad7d0d..b25e6bad 100644 --- a/packages/ciphernode/keyshare/Cargo.toml +++ b/packages/ciphernode/keyshare/Cargo.toml @@ -9,3 +9,5 @@ enclave-core = { path = "../core" } fhe = { path = "../fhe" } actix = { workspace = true } anyhow = { workspace = true } +serde = { workspace = true } +async-trait = { workspace = true } diff --git a/packages/ciphernode/keyshare/src/keyshare.rs b/packages/ciphernode/keyshare/src/keyshare.rs index 98e0cb94..ff680da0 100644 --- a/packages/ciphernode/keyshare/src/keyshare.rs +++ b/packages/ciphernode/keyshare/src/keyshare.rs @@ -1,17 +1,20 @@ use actix::prelude::*; -use anyhow::{anyhow, Context, Result}; -use data::{DataStore, Get, Insert}; +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use data::{Checkpoint, DataStore, FromSnapshotWithParams, Snapshot}; use enclave_core::{ - CiphernodeSelected, CiphertextOutputPublished, DecryptionshareCreated, Die, EnclaveErrorType, - EnclaveEvent, EventBus, FromError, KeyshareCreated, + BusError, CiphernodeSelected, CiphertextOutputPublished, DecryptionshareCreated, Die, + EnclaveErrorType, EnclaveEvent, EventBus, FromError, KeyshareCreated, }; use fhe::{DecryptCiphertext, Fhe}; +use serde::{Deserialize, Serialize}; use std::sync::Arc; pub struct Keyshare { fhe: Arc, - data: DataStore, + store: DataStore, bus: Addr, + secret: Option>, address: String, } @@ -19,17 +22,60 @@ impl Actor for Keyshare { type Context = actix::Context; } +pub struct KeyshareParams { + pub bus: Addr, + pub store: DataStore, + pub fhe: Arc, + pub address: String, +} + +#[derive(Serialize, Deserialize)] +pub struct KeyshareState { + secret: Option>, +} + impl Keyshare { - pub fn new(bus: Addr, data: DataStore, fhe: Arc, address: &str) -> Self { + pub fn new(params: KeyshareParams) -> Self { Self { - bus, - fhe, - data, - address: address.to_string(), + bus: params.bus, + fhe: params.fhe, + store: params.store, + secret: None, + address: params.address, + } + } +} + +impl Snapshot for Keyshare { + type Snapshot = KeyshareState; + + fn snapshot(&self) -> Self::Snapshot { + KeyshareState { + secret: self.secret.clone(), } } } +impl Checkpoint for Keyshare { + fn get_store(&self) -> DataStore { + self.store.clone() + } +} + +#[async_trait] +impl FromSnapshotWithParams for Keyshare { + type Params = KeyshareParams; + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + Ok(Self { + bus: params.bus, + fhe: params.fhe, + store: params.store, + secret: snapshot.secret, + address: params.address, + }) + } +} + impl Handler for Keyshare { type Result = (); @@ -50,52 +96,66 @@ impl Handler for Keyshare { let CiphernodeSelected { e3_id, .. } = event; // generate keyshare - let Ok((sk, pubkey)) = self.fhe.generate_keyshare() else { + let Ok((secret, pubkey)) = self.fhe.generate_keyshare() else { self.bus.do_send(EnclaveEvent::from_error( EnclaveErrorType::KeyGeneration, - anyhow!("Error creating Keyshare"), + anyhow!("Error creating Keyshare for {e3_id}"), )); return; }; - // TODO: decrypt from FHE actor - // save encrypted key against e3_id/sk - // reencrypt secretkey locally with env var - this is so we don't have to serialize a secret - // best practice would be as you boot up a node you enter in a configured password from - // which we derive a kdf which gets used to generate this key - self.data.write(format!("{e3_id}/sk"), sk); - - // save public key against e3_id/pk - self.data - .write(format!("{e3_id}/pk"), pubkey.clone()); + // Save secret on state + self.secret = Some(secret); - // broadcast the KeyshareCreated message - let event = EnclaveEvent::from(KeyshareCreated { + // Broadcast the KeyshareCreated message + self.bus.do_send(EnclaveEvent::from(KeyshareCreated { pubkey, e3_id, node: self.address.clone(), - }); - self.bus.do_send(event); + })); + + // Write the snapshot to the store + self.checkpoint() } } impl Handler for Keyshare { - type Result = ResponseFuture<()>; + type Result = (); fn handle( &mut self, event: CiphertextOutputPublished, _: &mut actix::Context, ) -> Self::Result { - let fhe = self.fhe.clone(); - let data = self.data.clone(); - let bus = self.bus.clone(); - let address = self.address.clone(); - Box::pin(async move { - on_decryption_requested(fhe, data, bus, event, address) - .await - .unwrap() - }) + let CiphertextOutputPublished { + e3_id, + ciphertext_output, + } = event; + + let Some(secret) = &self.secret else { + self.bus.err( + EnclaveErrorType::Decryption, + anyhow!("secret not found on Keyshare for e3_id {e3_id}"), + ); + return; + }; + + let Ok(decryption_share) = self.fhe.decrypt_ciphertext(DecryptCiphertext { + ciphertext: ciphertext_output.clone(), + unsafe_secret: secret.to_vec(), + }) else { + self.bus.err( + EnclaveErrorType::Decryption, + anyhow!("error decrypting ciphertext: {:?}", ciphertext_output), + ); + return; + }; + + self.bus.do_send(EnclaveEvent::from(DecryptionshareCreated { + e3_id, + decryption_share, + node: self.address.clone(), + })); } } @@ -105,40 +165,3 @@ impl Handler for Keyshare { ctx.stop() } } - -async fn on_decryption_requested( - fhe: Arc, - data: DataStore, - bus: Addr, - event: CiphertextOutputPublished, - address: String, -) -> Result<()> { - let CiphertextOutputPublished { - e3_id, - ciphertext_output, - } = event; - - // get secret key by id from data - let Some(unsafe_secret) = data.read(&format!("{e3_id}/sk")).await? else { - return Err(anyhow::anyhow!("Secret key not stored for {}", e3_id)); - }; - - println!("\n\nDECRYPTING!\n\n"); - - let decryption_share = fhe - .decrypt_ciphertext(DecryptCiphertext { - ciphertext: ciphertext_output, - unsafe_secret, - }) - .context("error decrypting ciphertext")?; - - let event = EnclaveEvent::from(DecryptionshareCreated { - e3_id, - decryption_share, - node: address, - }); - - bus.do_send(event); - - Ok(()) -} diff --git a/packages/ciphernode/router/Cargo.toml b/packages/ciphernode/router/Cargo.toml index 286095d8..fa6da46a 100644 --- a/packages/ciphernode/router/Cargo.toml +++ b/packages/ciphernode/router/Cargo.toml @@ -14,3 +14,4 @@ aggregator = { path = "../aggregator" } anyhow = { workspace = true } serde = { workspace = true } bincode = { workspace = true } +async-trait = { workspace = true } diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index c90b5f15..0cba9f02 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -1,5 +1,5 @@ use enclave_core::{E3Requested, EnclaveEvent, Seed}; - +use data::WithPrefix; use super::EventHook; #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] @@ -25,14 +25,15 @@ impl CommitteeMetaFactory { .. } = data; - ctx.set_meta( - &format!("//meta/{e3_id}"), - CommitteeMeta { - threshold_m, - seed, - src_chain_id, - }, - ); + // Meta doesn't implement Checkpoint so we are going to store it manually + let meta_id = format!("//meta/{e3_id}"); + let meta = CommitteeMeta { + threshold_m, + seed, + src_chain_id, + }; + ctx.get_store().at(&meta_id).write(meta.clone()); + ctx.set_meta(meta); }) } } diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index 92441a52..387ec68b 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -4,14 +4,18 @@ use actix::{Actor, Addr, Context, Handler, Recipient}; use aggregator::PlaintextAggregator; use aggregator::PublicKeyAggregator; use anyhow::*; +use async_trait::async_trait; +use data::Checkpoint; use data::DataStore; -use data::Get; -use data::Insert; +use data::FromSnapshotWithParams; +use data::Snapshot; use data::WithPrefix; use enclave_core::E3RequestComplete; use enclave_core::{E3id, EnclaveEvent, EventBus, Subscribe}; use fhe::Fhe; use keyshare::Keyshare; +use serde::Deserialize; +use serde::Serialize; use std::collections::HashSet; use std::{collections::HashMap, sync::Arc}; @@ -37,23 +41,40 @@ impl EventBuffer { /// Context that is set to each event hook. Hooks can use this context to gather dependencies if /// they need to instantiate struct instances or actors. pub struct E3RequestContext { + e3_id: E3id, keyshare: Option>, fhe: Option>, plaintext: Option>, publickey: Option>, meta: Option, + store: DataStore, +} + +#[derive(Serialize, Deserialize)] +pub struct E3RequestContextSnapshot { + e3_id: E3id, + keyshare: Option, + fhe: Option, + plaintext: Option, + publickey: Option, + meta: Option, +} + +pub struct E3RequestContextParams { pub store: DataStore, + pub e3_id: E3id, } impl E3RequestContext { - fn from_store(store: DataStore) -> Self { + pub fn from_params(params: E3RequestContextParams) -> Self { Self { - keyshare: None, + e3_id: params.e3_id, + store: params.store, fhe: None, + keyshare: None, + meta: None, plaintext: None, publickey: None, - meta: None, - store, } } @@ -88,47 +109,37 @@ impl E3RequestContext { } /// Accept a DataStore ID and a Keystore actor address - pub fn set_keyshare(&mut self, id: &str, value: Addr) -> Result<()> { - DataStore::ensure_root_id(id)?; + pub fn set_keyshare(&mut self, value: Addr) -> Result<()> { self.keyshare = Some(value); - self.store.write("keyshare", id.into()); + self.checkpoint(); Ok(()) } /// Accept a DataStore ID and a Keystore actor address - pub fn set_plaintext(&mut self, id: &str, value: Addr) -> Result<()> { - DataStore::ensure_root_id(id)?; + pub fn set_plaintext(&mut self, value: Addr) -> Result<()> { self.plaintext = Some(value); - self.store.write("plaintext", id.into()); + self.checkpoint(); Ok(()) } /// Accept a DataStore ID and a Keystore actor address - pub fn set_publickey(&mut self, id: &str, value: Addr) -> Result<()> { - DataStore::ensure_root_id(id)?; + pub fn set_publickey(&mut self, value: Addr) -> Result<()> { self.publickey = Some(value); - self.store.write("publickey", id.into()); + self.checkpoint(); Ok(()) } /// Accept a DataStore ID and an Arc instance of the Fhe wrapper - pub fn set_fhe(&mut self, id: &str, value: Arc) -> Result<()> { - DataStore::ensure_root_id(id)?; + pub fn set_fhe(&mut self, value: Arc) -> Result<()> { self.fhe = Some(value.clone()); - self.store.write("fhe", id.into()); - // TODO: should non actors store themselves? - self.store.write(id, value.to_bytes()?); + self.checkpoint(); Ok(()) } /// Accept a Datastore ID and a metadata object - pub fn set_meta(&mut self, id: &str, value: CommitteeMeta) -> Result<()> { - DataStore::ensure_root_id(id)?; + pub fn set_meta(&mut self, value: CommitteeMeta) -> Result<()> { self.meta = Some(value.clone()); - self.store.write("meta", id.into()); - // TODO: should non actors store themselves? - self.store - .write(id, bincode::serialize(&value)?); + self.checkpoint(); Ok(()) } @@ -151,6 +162,64 @@ impl E3RequestContext { pub fn get_meta(&self) -> Option<&CommitteeMeta> { self.meta.as_ref() } + + pub fn get_store(&self) -> DataStore { + self.store.clone() + } +} + +#[async_trait] +impl Snapshot for E3RequestContext { + type Snapshot = E3RequestContextSnapshot; + + fn snapshot(&self) -> Self::Snapshot { + let e3_id = self.e3_id.clone(); + let meta = self.meta.as_ref().map(|_| format!("//meta/{e3_id}")); + let fhe = self.fhe.as_ref().map(|_| format!("//fhe/{e3_id}")); + let publickey = self + .publickey + .as_ref() + .map(|_| format!("//publickey/{e3_id}")); + let plaintext = self + .plaintext + .as_ref() + .map(|_| format!("//plaintext/{e3_id}")); + let keyshare = self + .keyshare + .as_ref() + .map(|_| format!("//keyshare/{e3_id}")); + + Self::Snapshot { + e3_id, + meta, + fhe, + publickey, + plaintext, + keyshare, + } + } +} + +#[async_trait] +impl FromSnapshotWithParams for E3RequestContext { + type Params = E3RequestContextParams; + async fn from_snapshot(params: Self::Params, _: Self::Snapshot) -> Result { + Ok(Self { + e3_id: params.e3_id, + store: params.store, + fhe: None, + keyshare: None, + meta: None, + plaintext: None, + publickey: None, + }) + } +} + +impl Checkpoint for E3RequestContext { + fn get_store(&self) -> DataStore { + self.store.clone() + } } /// Format of the hook that needs to be passed to E3RequestRouter @@ -200,10 +269,12 @@ impl Handler for E3RequestRouter { return; } - let context = self - .contexts - .entry(e3_id.clone()) - .or_insert_with(|| E3RequestContext::from_store(self.store.clone())); + let context = self.contexts.entry(e3_id.clone()).or_insert_with(|| { + E3RequestContext::from_params(E3RequestContextParams { + e3_id: e3_id.clone(), + store: self.store.at(&format!("//context/{e3_id}")), + }) + }); for hook in &mut self.hooks { hook(context, msg.clone()); diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index 9c2345b1..ebfc6891 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -1,10 +1,13 @@ use crate::EventHook; use actix::{Actor, Addr}; -use aggregator::{PlaintextAggregator, PublicKeyAggregator}; -use data::{DataStore, WithPrefix}; +use aggregator::{ + PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorState, PublicKeyAggregator, + PublicKeyAggregatorParams, PublicKeyAggregatorState, +}; +use data::{Snapshot, WithPrefix}; use enclave_core::{E3Requested, EnclaveEvent, EventBus}; use fhe::{Fhe, SharedRng}; -use keyshare::Keyshare; +use keyshare::{Keyshare, KeyshareParams}; use sortition::Sortition; use std::sync::Arc; @@ -25,11 +28,12 @@ impl LazyFhe { .. } = data; - // Set the FHE instance passing in the instance id - let _ = ctx.set_fhe( - &format!("//fhe/{e3_id}"), - Arc::new(Fhe::from_encoded(¶ms, seed, rng.clone()).unwrap()), - ); + // FHE doesn't implement Checkpoint so we are going to store it manually + let fhe_id = format!("//fhe/{e3_id}"); + let fhe = Arc::new(Fhe::from_encoded(¶ms, seed, rng.clone()).unwrap()); + ctx.get_store().at(&fhe_id).write(fhe.snapshot()); + + let _ = ctx.set_fhe(fhe); }) } } @@ -50,16 +54,15 @@ impl LazyKeyshare { let e3_id = data.e3_id; - let ks_id = &format!("//keystore/{e3_id}"); + let ks_id = format!("//keystore/{e3_id}"); let _ = ctx.set_keyshare( - ks_id, - Keyshare::new( - bus.clone(), - ctx.store.clone().base(ks_id), - fhe.clone(), - &address, - ) + Keyshare::new(KeyshareParams { + bus: bus.clone(), + store: ctx.get_store().at(&ks_id), + fhe: fhe.clone(), + address: address.clone(), + }) .start(), ); }) @@ -86,16 +89,20 @@ impl LazyPlaintextAggregator { let id = &format!("//plaintext/{e3_id}"); let _ = ctx.set_plaintext( - id, PlaintextAggregator::new( - fhe.clone(), - bus.clone(), - sortition.clone(), - e3_id, - meta.threshold_m, - meta.seed, - data.ciphertext_output, - meta.src_chain_id, + PlaintextAggregatorParams { + fhe: fhe.clone(), + bus: bus.clone(), + store: ctx.get_store().at(id), + sortition: sortition.clone(), + e3_id, + src_chain_id: meta.src_chain_id, + }, + PlaintextAggregatorState::init( + meta.threshold_m, + meta.seed, + data.ciphertext_output, + ), ) .start(), ); @@ -125,15 +132,16 @@ impl LazyPublicKeyAggregator { let id = &format!("//publickey/{e3_id}"); let _ = ctx.set_publickey( - id, PublicKeyAggregator::new( - fhe.clone(), - bus.clone(), - sortition.clone(), - e3_id, - meta.threshold_m, - meta.seed, - meta.src_chain_id, + PublicKeyAggregatorParams { + fhe: fhe.clone(), + bus: bus.clone(), + store: ctx.get_store().at(id), + sortition: sortition.clone(), + e3_id, + src_chain_id: meta.src_chain_id, + }, + PublicKeyAggregatorState::init(meta.threshold_m, meta.seed), ) .start(), ); From 666ae206ad7691858e1e4fc503c0e962c06e7f2e Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 15 Oct 2024 19:20:10 +1100 Subject: [PATCH 05/49] Formatting --- packages/ciphernode/enclave_node/src/ciphernode.rs | 5 +---- packages/ciphernode/router/src/committee_meta.rs | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 09a0af26..14e3b1ca 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -81,10 +81,7 @@ impl MainCiphernode { let e3_manager = E3RequestRouter::builder(bus.clone(), data.clone()) .add_hook(LazyFhe::create(rng)) - .add_hook(LazyKeyshare::create( - bus.clone(), - &address.to_string(), - )) + .add_hook(LazyKeyshare::create(bus.clone(), &address.to_string())) .build(); let (p2p_addr, join_handle) = diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index 0cba9f02..8d4b4e9d 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -1,6 +1,6 @@ -use enclave_core::{E3Requested, EnclaveEvent, Seed}; -use data::WithPrefix; use super::EventHook; +use data::WithPrefix; +use enclave_core::{E3Requested, EnclaveEvent, Seed}; #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct CommitteeMeta { @@ -32,7 +32,7 @@ impl CommitteeMetaFactory { seed, src_chain_id, }; - ctx.get_store().at(&meta_id).write(meta.clone()); + ctx.get_store().at(&meta_id).write(meta.clone()); ctx.set_meta(meta); }) } From bc47b62020e7ccc05054ebcbf8a23888d25275fd Mon Sep 17 00:00:00 2001 From: ryardley Date: Tue, 15 Oct 2024 22:25:30 +1100 Subject: [PATCH 06/49] Setup features --- .../ciphernode/enclave_node/src/aggregator.rs | 8 +- .../ciphernode/enclave_node/src/ciphernode.rs | 6 +- .../ciphernode/router/src/committee_meta.rs | 59 ++-- .../router/src/e3_request_router.rs | 57 ++-- packages/ciphernode/router/src/hooks.rs | 286 ++++++++++-------- .../tests/test_aggregation_and_decryption.rs | 12 +- 6 files changed, 244 insertions(+), 184 deletions(-) diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 78ea1d3d..412117bb 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -9,7 +9,7 @@ use logger::SimpleLogger; use p2p::P2p; use rand::SeedableRng; use rand_chacha::rand_core::OsRng; -use router::{E3RequestRouter, LazyFhe, LazyPlaintextAggregator, LazyPublicKeyAggregator}; +use router::{E3RequestRouter, FheFeature, PlaintextAggregatorFeature, PublicKeyAggregatorFeature}; use sortition::Sortition; use std::sync::{Arc, Mutex}; use test_helpers::{PlaintextWriter, PublicKeyWriter}; @@ -84,12 +84,12 @@ impl MainAggregator { } let e3_manager = E3RequestRouter::builder(bus.clone(), store) - .add_hook(LazyFhe::create(rng)) - .add_hook(LazyPublicKeyAggregator::create( + .add_feature(FheFeature::create(rng)) + .add_feature(PublicKeyAggregatorFeature::create( bus.clone(), sortition.clone(), )) - .add_hook(LazyPlaintextAggregator::create( + .add_feature(PlaintextAggregatorFeature::create( bus.clone(), sortition.clone(), )) diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 14e3b1ca..0ac7917b 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -8,7 +8,7 @@ use logger::SimpleLogger; use p2p::P2p; use rand::SeedableRng; use rand_chacha::rand_core::OsRng; -use router::{CiphernodeSelector, E3RequestRouter, LazyFhe, LazyKeyshare}; +use router::{CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature}; use sortition::Sortition; use std::sync::{Arc, Mutex}; use tokio::task::JoinHandle; @@ -80,8 +80,8 @@ impl MainCiphernode { } let e3_manager = E3RequestRouter::builder(bus.clone(), data.clone()) - .add_hook(LazyFhe::create(rng)) - .add_hook(LazyKeyshare::create(bus.clone(), &address.to_string())) + .add_feature(FheFeature::create(rng)) + .add_feature(KeyshareFeature::create(bus.clone(), &address.to_string())) .build(); let (p2p_addr, join_handle) = diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index 8d4b4e9d..2a0aa071 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -1,4 +1,6 @@ -use super::EventHook; +use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot}; + +use async_trait::async_trait; use data::WithPrefix; use enclave_core::{E3Requested, EnclaveEvent, Seed}; @@ -9,31 +11,38 @@ pub struct CommitteeMeta { pub src_chain_id: u64, } -pub struct CommitteeMetaFactory; +pub struct CommitteMetaFeature; -impl CommitteeMetaFactory { - pub fn create() -> EventHook { - Box::new(move |ctx, evt| { - let EnclaveEvent::E3Requested { data, .. } = evt else { - return; - }; - let E3Requested { - threshold_m, - seed, - src_chain_id, - e3_id, - .. - } = data; +impl CommitteMetaFeature { + pub fn create() -> Box { + Box::new(Self {}) + } +} - // Meta doesn't implement Checkpoint so we are going to store it manually - let meta_id = format!("//meta/{e3_id}"); - let meta = CommitteeMeta { - threshold_m, - seed, - src_chain_id, - }; - ctx.get_store().at(&meta_id).write(meta.clone()); - ctx.set_meta(meta); - }) +#[async_trait] +impl E3Feature for CommitteMetaFeature { + fn event(&self, ctx: &mut crate::E3RequestContext, event: &EnclaveEvent) { + let EnclaveEvent::E3Requested { data, .. } = event else { + return; + }; + let E3Requested { + threshold_m, + seed, + src_chain_id, + e3_id, + .. + } = data.clone(); + + // Meta doesn't implement Checkpoint so we are going to store it manually + let meta_id = format!("//meta/{e3_id}"); + let meta = CommitteeMeta { + threshold_m, + seed, + src_chain_id, + }; + ctx.get_store().at(&meta_id).write(meta.clone()); + let _ = ctx.set_meta(meta); } + + async fn hydrate(&self, _ctx: &mut E3RequestContext, _snapshot: &E3RequestContextSnapshot) {} } diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index 387ec68b..5cf31b69 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -1,5 +1,6 @@ +use crate::CommitteMetaFeature; + use super::CommitteeMeta; -use crate::CommitteeMetaFactory; use actix::{Actor, Addr, Context, Handler, Recipient}; use aggregator::PlaintextAggregator; use aggregator::PublicKeyAggregator; @@ -41,28 +42,28 @@ impl EventBuffer { /// Context that is set to each event hook. Hooks can use this context to gather dependencies if /// they need to instantiate struct instances or actors. pub struct E3RequestContext { - e3_id: E3id, - keyshare: Option>, - fhe: Option>, - plaintext: Option>, - publickey: Option>, - meta: Option, - store: DataStore, + pub e3_id: E3id, + pub keyshare: Option>, + pub fhe: Option>, + pub plaintext: Option>, + pub publickey: Option>, + pub meta: Option, + pub store: DataStore, } #[derive(Serialize, Deserialize)] pub struct E3RequestContextSnapshot { - e3_id: E3id, - keyshare: Option, - fhe: Option, - plaintext: Option, - publickey: Option, - meta: Option, + pub keyshare: Option, + pub fhe: Option, + pub plaintext: Option, + pub publickey: Option, + pub meta: Option, } pub struct E3RequestContextParams { pub store: DataStore, pub e3_id: E3id, + // pub hooks: Option> } impl E3RequestContext { @@ -190,7 +191,6 @@ impl Snapshot for E3RequestContext { .map(|_| format!("//keyshare/{e3_id}")); Self::Snapshot { - e3_id, meta, fhe, publickey, @@ -204,7 +204,7 @@ impl Snapshot for E3RequestContext { impl FromSnapshotWithParams for E3RequestContext { type Params = E3RequestContextParams; async fn from_snapshot(params: Self::Params, _: Self::Snapshot) -> Result { - Ok(Self { + let ctx = Self { e3_id: params.e3_id, store: params.store, fhe: None, @@ -212,7 +212,8 @@ impl FromSnapshotWithParams for E3RequestContext { meta: None, plaintext: None, publickey: None, - }) + }; + Ok(ctx) } } @@ -223,7 +224,15 @@ impl Checkpoint for E3RequestContext { } /// Format of the hook that needs to be passed to E3RequestRouter -pub type EventHook = Box; +// pub type EventHook = Box; +// pub type Hydrator = Box; +// pub type E3Feature = (EventHook, Hydrator); + +#[async_trait] +pub trait E3Feature { + fn event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent); + async fn hydrate(&self, ctx: &mut E3RequestContext, snapshot: &E3RequestContextSnapshot); +} /// E3RequestRouter will register hooks that receive an E3_id specific context. After hooks /// have run e3_id specific messages are forwarded to all instances on the context. This enables @@ -234,7 +243,7 @@ pub type EventHook = Box; pub struct E3RequestRouter { contexts: HashMap, completed: HashSet, - hooks: Vec, + hooks: Vec>, buffer: EventBuffer, bus: Addr, store: DataStore, @@ -249,7 +258,7 @@ impl E3RequestRouter { }; // Everything needs the committe meta factory so adding it here by default - builder.add_hook(CommitteeMetaFactory::create()) + builder.add_feature(CommitteMetaFeature::create()) } } @@ -276,8 +285,8 @@ impl Handler for E3RequestRouter { }) }); - for hook in &mut self.hooks { - hook(context, msg.clone()); + for feature in &mut self.hooks { + feature.event(context, &msg); } context.forward_message(&msg, &mut self.buffer); @@ -308,11 +317,11 @@ impl Handler for E3RequestRouter { /// Builder for E3RequestRouter pub struct E3RequestRouterBuilder { pub bus: Addr, - pub hooks: Vec, + pub hooks: Vec>, pub store: DataStore, } impl E3RequestRouterBuilder { - pub fn add_hook(mut self, listener: EventHook) -> Self { + pub fn add_feature(mut self, listener: Box) -> Self { self.hooks.push(listener); self } diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index ebfc6891..95a2fa55 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -1,9 +1,10 @@ -use crate::EventHook; +use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot}; use actix::{Actor, Addr}; use aggregator::{ PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorState, PublicKeyAggregator, PublicKeyAggregatorParams, PublicKeyAggregatorState, }; +use async_trait::async_trait; use data::{Snapshot, WithPrefix}; use enclave_core::{E3Requested, EnclaveEvent, EventBus}; use fhe::{Fhe, SharedRng}; @@ -11,140 +12,181 @@ use keyshare::{Keyshare, KeyshareParams}; use sortition::Sortition; use std::sync::Arc; -pub struct LazyFhe; - -impl LazyFhe { - pub fn create(rng: SharedRng) -> EventHook { - Box::new(move |ctx, evt| { - // Saving the fhe on Committee Requested - let EnclaveEvent::E3Requested { data, .. } = evt else { - return; - }; - - let E3Requested { - params, - seed, - e3_id, - .. - } = data; - - // FHE doesn't implement Checkpoint so we are going to store it manually - let fhe_id = format!("//fhe/{e3_id}"); - let fhe = Arc::new(Fhe::from_encoded(¶ms, seed, rng.clone()).unwrap()); - ctx.get_store().at(&fhe_id).write(fhe.snapshot()); - - let _ = ctx.set_fhe(fhe); - }) +pub struct FheFeature { + rng: SharedRng, +} + +impl FheFeature { + pub fn create(rng: SharedRng) -> Box { + Box::new(Self { rng }) + } +} + +#[async_trait] +impl E3Feature for FheFeature { + fn event(&self, ctx: &mut crate::E3RequestContext, evt: &EnclaveEvent) { + // Saving the fhe on Committee Requested + let EnclaveEvent::E3Requested { data, .. } = evt else { + return; + }; + + let E3Requested { + params, + seed, + e3_id, + .. + } = data.clone(); + + // FHE doesn't implement Checkpoint so we are going to store it manually + let fhe_id = format!("//fhe/{e3_id}"); + let fhe = Arc::new(Fhe::from_encoded(¶ms, seed, self.rng.clone()).unwrap()); + ctx.get_store().at(&fhe_id).write(fhe.snapshot()); + + let _ = ctx.set_fhe(fhe); } + async fn hydrate(&self, _ctx: &mut E3RequestContext, _snapshot: &E3RequestContextSnapshot) {} } -pub struct LazyKeyshare; -impl LazyKeyshare { - pub fn create(bus: Addr, address: &str) -> EventHook { - let address = address.to_string(); - Box::new(move |ctx, evt| { - // Save Ciphernode on CiphernodeSelected - let EnclaveEvent::CiphernodeSelected { data, .. } = evt else { - return; - }; +pub struct KeyshareFeature { + bus: Addr, + address: String, +} - let Some(fhe) = ctx.get_fhe() else { - return; - }; +impl KeyshareFeature { + pub fn create(bus: Addr, address: &str) -> Box { + Box::new(Self { + bus, + address: address.to_owned(), + }) + } +} - let e3_id = data.e3_id; +#[async_trait] +impl E3Feature for KeyshareFeature { + fn event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { + // Save Ciphernode on CiphernodeSelected + let EnclaveEvent::CiphernodeSelected { data, .. } = evt else { + return; + }; + + let Some(fhe) = ctx.get_fhe() else { + return; + }; + + let e3_id = data.clone().e3_id; + + let ks_id = format!("//keystore/{e3_id}"); + + let _ = ctx.set_keyshare( + Keyshare::new(KeyshareParams { + bus: self.bus.clone(), + store: ctx.get_store().at(&ks_id), + fhe: fhe.clone(), + address: self.address.clone(), + }) + .start(), + ); + } - let ks_id = format!("//keystore/{e3_id}"); + async fn hydrate(&self, _ctx: &mut E3RequestContext, _snapshot: &E3RequestContextSnapshot) {} +} - let _ = ctx.set_keyshare( - Keyshare::new(KeyshareParams { - bus: bus.clone(), - store: ctx.get_store().at(&ks_id), +pub struct PlaintextAggregatorFeature { + bus: Addr, + sortition: Addr, +} +impl PlaintextAggregatorFeature { + pub fn create(bus: Addr, sortition: Addr) -> Box { + Box::new(Self { bus, sortition }) + } +} + +#[async_trait] +impl E3Feature for PlaintextAggregatorFeature { + fn event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { + // Save plaintext aggregator + let EnclaveEvent::CiphertextOutputPublished { data, .. } = evt else { + return; + }; + let Some(fhe) = ctx.get_fhe() else { + return; + }; + let Some(ref meta) = ctx.get_meta() else { + return; + }; + + let e3_id = data.e3_id.clone(); + + let id = &format!("//plaintext/{e3_id}"); + + let _ = ctx.set_plaintext( + PlaintextAggregator::new( + PlaintextAggregatorParams { fhe: fhe.clone(), - address: address.clone(), - }) - .start(), - ); - }) + bus: self.bus.clone(), + store: ctx.get_store().at(id), + sortition: self.sortition.clone(), + e3_id, + src_chain_id: meta.src_chain_id, + }, + PlaintextAggregatorState::init( + meta.threshold_m, + meta.seed, + data.ciphertext_output.clone(), + ), + ) + .start(), + ); } + async fn hydrate(&self, _ctx: &mut E3RequestContext, _snapshot: &E3RequestContextSnapshot) {} } -pub struct LazyPlaintextAggregator; -impl LazyPlaintextAggregator { - pub fn create(bus: Addr, sortition: Addr) -> EventHook { - Box::new(move |ctx, evt| { - // Save plaintext aggregator - let EnclaveEvent::CiphertextOutputPublished { data, .. } = evt else { - return; - }; - let Some(fhe) = ctx.get_fhe() else { - return; - }; - let Some(ref meta) = ctx.get_meta() else { - return; - }; - - let e3_id = data.e3_id; - - let id = &format!("//plaintext/{e3_id}"); - - let _ = ctx.set_plaintext( - PlaintextAggregator::new( - PlaintextAggregatorParams { - fhe: fhe.clone(), - bus: bus.clone(), - store: ctx.get_store().at(id), - sortition: sortition.clone(), - e3_id, - src_chain_id: meta.src_chain_id, - }, - PlaintextAggregatorState::init( - meta.threshold_m, - meta.seed, - data.ciphertext_output, - ), - ) - .start(), - ); - }) +pub struct PublicKeyAggregatorFeature { + bus: Addr, + sortition: Addr, +} + +impl PublicKeyAggregatorFeature { + pub fn create(bus: Addr, sortition: Addr) -> Box { + Box::new(Self { bus, sortition }) } } -pub struct LazyPublicKeyAggregator; -impl LazyPublicKeyAggregator { - pub fn create(bus: Addr, sortition: Addr) -> EventHook { - Box::new(move |ctx, evt| { - // Saving the publickey aggregator with deps on E3Requested - let EnclaveEvent::E3Requested { data, .. } = evt else { - return; - }; - - let Some(fhe) = ctx.get_fhe() else { - println!("fhe was not on ctx"); - return; - }; - let Some(ref meta) = ctx.get_meta() else { - println!("meta was not on ctx"); - return; - }; - - let e3_id = data.e3_id; - let id = &format!("//publickey/{e3_id}"); - - let _ = ctx.set_publickey( - PublicKeyAggregator::new( - PublicKeyAggregatorParams { - fhe: fhe.clone(), - bus: bus.clone(), - store: ctx.get_store().at(id), - sortition: sortition.clone(), - e3_id, - src_chain_id: meta.src_chain_id, - }, - PublicKeyAggregatorState::init(meta.threshold_m, meta.seed), - ) - .start(), - ); - }) +#[async_trait] +impl E3Feature for PublicKeyAggregatorFeature { + fn event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { + // Saving the publickey aggregator with deps on E3Requested + let EnclaveEvent::E3Requested { data, .. } = evt else { + return; + }; + + let Some(fhe) = ctx.get_fhe() else { + println!("fhe was not on ctx"); + return; + }; + let Some(ref meta) = ctx.get_meta() else { + println!("meta was not on ctx"); + return; + }; + + let e3_id = data.e3_id.clone(); + let id = &format!("//publickey/{e3_id}"); + + let _ = ctx.set_publickey( + PublicKeyAggregator::new( + PublicKeyAggregatorParams { + fhe: fhe.clone(), + bus: self.bus.clone(), + store: ctx.get_store().at(id), + sortition: self.sortition.clone(), + e3_id, + src_chain_id: meta.src_chain_id, + }, + PublicKeyAggregatorState::init(meta.threshold_m, meta.seed), + ) + .start(), + ); } + + async fn hydrate(&self, _ctx: &mut E3RequestContext, _snapshot: &E3RequestContextSnapshot) {} } diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 4232a2ae..b53db40f 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -8,8 +8,8 @@ use fhe::{setup_crp_params, ParamsWithCrp, SharedRng}; use logger::SimpleLogger; use p2p::P2p; use router::{ - CiphernodeSelector, E3RequestRouter, LazyFhe, LazyKeyshare, LazyPlaintextAggregator, - LazyPublicKeyAggregator, + CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, PlaintextAggregatorFeature, + PublicKeyAggregatorFeature, }; use sortition::Sortition; @@ -39,16 +39,16 @@ async fn setup_local_ciphernode(bus: Addr, rng: SharedRng, logging: bo CiphernodeSelector::attach(bus.clone(), sortition.clone(), addr); E3RequestRouter::builder(bus.clone(), store) - .add_hook(LazyFhe::create(rng)) - .add_hook(LazyPublicKeyAggregator::create( + .add_feature(FheFeature::create(rng)) + .add_feature(PublicKeyAggregatorFeature::create( bus.clone(), sortition.clone(), )) - .add_hook(LazyPlaintextAggregator::create( + .add_feature(PlaintextAggregatorFeature::create( bus.clone(), sortition.clone(), )) - .add_hook(LazyKeyshare::create(bus.clone(), addr)) + .add_feature(KeyshareFeature::create(bus.clone(), addr)) .build(); SimpleLogger::attach(addr, bus.clone()); From 957267d1609de04e5e2001eb1951846313e11012 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 16 Oct 2024 00:24:04 +1100 Subject: [PATCH 07/49] Add hydration --- packages/ciphernode/data/src/data_store.rs | 2 +- .../ciphernode/enclave_node/src/aggregator.rs | 2 +- .../ciphernode/enclave_node/src/ciphernode.rs | 2 +- .../ciphernode/router/src/committee_meta.rs | 10 +- .../router/src/e3_request_router.rs | 157 ++++++++++++++---- packages/ciphernode/router/src/hooks.rs | 154 ++++++++++++++++- .../tests/test_aggregation_and_decryption.rs | 17 +- 7 files changed, 294 insertions(+), 50 deletions(-) diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index aa58aabc..72c83511 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -66,7 +66,7 @@ pub trait Checkpoint: Snapshot { #[async_trait] pub trait FromSnapshotWithParams: Snapshot { - type Params; + type Params:Send + 'static; /// Return an instance of the persistable object at the state given by the snapshot /// This method is async because there may be subobjects that require hydration from the store diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 412117bb..9f2c149e 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -93,7 +93,7 @@ impl MainAggregator { bus.clone(), sortition.clone(), )) - .build(); + .build().await?; let (p2p_addr, join_handle) = P2p::spawn_libp2p(bus.clone()).expect("Failed to setup libp2p"); diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 0ac7917b..6cd6e2d4 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -82,7 +82,7 @@ impl MainCiphernode { let e3_manager = E3RequestRouter::builder(bus.clone(), data.clone()) .add_feature(FheFeature::create(rng)) .add_feature(KeyshareFeature::create(bus.clone(), &address.to_string())) - .build(); + .build().await?; let (p2p_addr, join_handle) = P2p::spawn_libp2p(bus.clone()).expect("Failed to setup libp2p"); diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index 2a0aa071..6eee0497 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -1,5 +1,5 @@ use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot}; - +use anyhow::*; use async_trait::async_trait; use data::WithPrefix; use enclave_core::{E3Requested, EnclaveEvent, Seed}; @@ -44,5 +44,11 @@ impl E3Feature for CommitteMetaFeature { let _ = ctx.set_meta(meta); } - async fn hydrate(&self, _ctx: &mut E3RequestContext, _snapshot: &E3RequestContextSnapshot) {} + async fn hydrate( + &self, + _ctx: &mut E3RequestContext, + _snapshot: &E3RequestContextSnapshot, + ) -> Result<()> { + Ok(()) + } } diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index 5cf31b69..cc664461 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -18,6 +18,7 @@ use keyshare::Keyshare; use serde::Deserialize; use serde::Serialize; use std::collections::HashSet; +use std::pin::Pin; use std::{collections::HashMap, sync::Arc}; /// Helper class to buffer events for downstream instances incase events arrive in the wrong order @@ -63,7 +64,7 @@ pub struct E3RequestContextSnapshot { pub struct E3RequestContextParams { pub store: DataStore, pub e3_id: E3id, - // pub hooks: Option> + pub features: Arc>>, } impl E3RequestContext { @@ -110,38 +111,33 @@ impl E3RequestContext { } /// Accept a DataStore ID and a Keystore actor address - pub fn set_keyshare(&mut self, value: Addr) -> Result<()> { + pub fn set_keyshare(&mut self, value: Addr) { self.keyshare = Some(value); self.checkpoint(); - Ok(()) } /// Accept a DataStore ID and a Keystore actor address - pub fn set_plaintext(&mut self, value: Addr) -> Result<()> { + pub fn set_plaintext(&mut self, value: Addr) { self.plaintext = Some(value); self.checkpoint(); - Ok(()) } /// Accept a DataStore ID and a Keystore actor address - pub fn set_publickey(&mut self, value: Addr) -> Result<()> { + pub fn set_publickey(&mut self, value: Addr) { self.publickey = Some(value); self.checkpoint(); - Ok(()) } /// Accept a DataStore ID and an Arc instance of the Fhe wrapper - pub fn set_fhe(&mut self, value: Arc) -> Result<()> { + pub fn set_fhe(&mut self, value: Arc) { self.fhe = Some(value.clone()); self.checkpoint(); - Ok(()) } /// Accept a Datastore ID and a metadata object - pub fn set_meta(&mut self, value: CommitteeMeta) -> Result<()> { + pub fn set_meta(&mut self, value: CommitteeMeta) { self.meta = Some(value.clone()); self.checkpoint(); - Ok(()) } pub fn get_keyshare(&self) -> Option<&Addr> { @@ -203,8 +199,8 @@ impl Snapshot for E3RequestContext { #[async_trait] impl FromSnapshotWithParams for E3RequestContext { type Params = E3RequestContextParams; - async fn from_snapshot(params: Self::Params, _: Self::Snapshot) -> Result { - let ctx = Self { + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + let mut ctx = Self { e3_id: params.e3_id, store: params.store, fhe: None, @@ -213,6 +209,11 @@ impl FromSnapshotWithParams for E3RequestContext { plaintext: None, publickey: None, }; + + for feature in params.features.iter() { + feature.hydrate(&mut ctx, &snapshot).await?; + } + Ok(ctx) } } @@ -229,37 +230,60 @@ impl Checkpoint for E3RequestContext { // pub type E3Feature = (EventHook, Hydrator); #[async_trait] -pub trait E3Feature { +pub trait E3Feature: Send + Sync + 'static { fn event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent); - async fn hydrate(&self, ctx: &mut E3RequestContext, snapshot: &E3RequestContextSnapshot); + async fn hydrate( + &self, + ctx: &mut E3RequestContext, + snapshot: &E3RequestContextSnapshot, + ) -> Result<()>; } -/// E3RequestRouter will register hooks that receive an E3_id specific context. After hooks +/// E3RequestRouter will register features that receive an E3_id specific context. After features /// have run e3_id specific messages are forwarded to all instances on the context. This enables -/// hooks to lazily register instances that have the correct dependencies available per e3_id +/// features to lazily register instances that have the correct dependencies available per e3_id /// request -// TODO: setup typestate pattern so that we have to place hooks within correct order of +// TODO: setup typestate pattern so that we have to place features within correct order of // dependencies pub struct E3RequestRouter { contexts: HashMap, completed: HashSet, - hooks: Vec>, + features: Arc>>, buffer: EventBuffer, bus: Addr, store: DataStore, } +pub struct E3RequestRouterParams { + features: Arc>>, + bus: Addr, + store: DataStore, +} + impl E3RequestRouter { pub fn builder(bus: Addr, store: DataStore) -> E3RequestRouterBuilder { let builder = E3RequestRouterBuilder { bus, - hooks: vec![], + features: vec![], store, }; // Everything needs the committe meta factory so adding it here by default builder.add_feature(CommitteMetaFeature::create()) } + + pub fn from_params(params: E3RequestRouterParams) -> Self { + Self { + features: params.features, + bus: params.bus.clone(), + store: params.store.clone(), + completed: HashSet::new(), + contexts: HashMap::new(), + buffer: EventBuffer { + buffer: HashMap::new(), + }, + } + } } impl Actor for E3RequestRouter { @@ -282,10 +306,11 @@ impl Handler for E3RequestRouter { E3RequestContext::from_params(E3RequestContextParams { e3_id: e3_id.clone(), store: self.store.at(&format!("//context/{e3_id}")), + features: self.features.clone(), }) }); - for feature in &mut self.hooks { + for feature in self.features.clone().iter() { feature.event(context, &msg); } @@ -311,38 +336,102 @@ impl Handler for E3RequestRouter { } _ => (), } + + self.checkpoint(); + } +} + +#[derive(Serialize, Deserialize)] +pub struct E3RequestRouterSnapshot { + contexts: Vec, + completed: HashSet, +} + +impl Snapshot for E3RequestRouter { + type Snapshot = E3RequestRouterSnapshot; + fn snapshot(&self) -> Self::Snapshot { + let contexts = self.contexts.keys().cloned().collect(); + let completed = self.completed.clone(); + + Self::Snapshot { + completed, + contexts, + } + } +} + +impl Checkpoint for E3RequestRouter { + fn get_store(&self) -> DataStore { + self.store.clone() + } +} + +#[async_trait] +impl FromSnapshotWithParams for E3RequestRouter { + type Params = E3RequestRouterParams; + + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + let mut contexts = HashMap::new(); + for e3_id in snapshot.contexts { + let Some(ctx_snapshot) = params.store.read(format!("//context/{e3_id}")).await? else { + continue; + }; + + contexts.insert( + e3_id.clone(), + E3RequestContext::from_snapshot( + E3RequestContextParams { + store: params.store.clone(), + e3_id: e3_id.clone(), + features: params.features.clone(), + }, + ctx_snapshot, + ) + .await?, + ); + } + + Ok(E3RequestRouter { + contexts, + completed: snapshot.completed, + features: params.features.into(), + buffer: EventBuffer::default(), + bus: params.bus, + store: params.store, + }) } } /// Builder for E3RequestRouter pub struct E3RequestRouterBuilder { pub bus: Addr, - pub hooks: Vec>, + pub features: Vec>, pub store: DataStore, } + impl E3RequestRouterBuilder { pub fn add_feature(mut self, listener: Box) -> Self { - self.hooks.push(listener); + self.features.push(listener); self } - pub fn build(self) -> Addr { - let e3r = E3RequestRouter { - contexts: HashMap::new(), - completed: HashSet::new(), - hooks: self.hooks, - buffer: EventBuffer::default(), + pub async fn build(self) -> Result> { + let snapshot: Option = self.store.at("").read("//router").await?; + let params = E3RequestRouterParams { + features: self.features.into(), bus: self.bus.clone(), + store: self.store, }; + let e3r = match snapshot { + Some(snapshot) => E3RequestRouter::from_snapshot(params, snapshot).await?, + None => E3RequestRouter::from_params(params), + }; + let addr = e3r.start(); self.bus .do_send(Subscribe::new("*", addr.clone().recipient())); - addr + Ok(addr) } - - // pub async fn hydrate(self) -> Addr { - // let store = self.store.base("//router"); - // } } diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index 95a2fa55..795a8053 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -4,8 +4,9 @@ use aggregator::{ PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorState, PublicKeyAggregator, PublicKeyAggregatorParams, PublicKeyAggregatorState, }; +use anyhow::Result; use async_trait::async_trait; -use data::{Snapshot, WithPrefix}; +use data::{FromSnapshotWithParams, Snapshot, WithPrefix}; use enclave_core::{E3Requested, EnclaveEvent, EventBus}; use fhe::{Fhe, SharedRng}; use keyshare::{Keyshare, KeyshareParams}; @@ -41,10 +42,29 @@ impl E3Feature for FheFeature { let fhe_id = format!("//fhe/{e3_id}"); let fhe = Arc::new(Fhe::from_encoded(¶ms, seed, self.rng.clone()).unwrap()); ctx.get_store().at(&fhe_id).write(fhe.snapshot()); - let _ = ctx.set_fhe(fhe); } - async fn hydrate(&self, _ctx: &mut E3RequestContext, _snapshot: &E3RequestContextSnapshot) {} + + async fn hydrate( + &self, + ctx: &mut E3RequestContext, + snapshot: &E3RequestContextSnapshot, + ) -> Result<()> { + // No ID on the snapshot -> bail + let Some(id) = snapshot.fhe.clone() else { + return Ok(()); + }; + + // No Snapshot returned from the store -> bail + let Some(snap) = ctx.store.at("").read(&id).await? else { + return Ok(()); + }; + + let value = Arc::new(Fhe::from_snapshot(self.rng.clone(), snap).await?); + ctx.set_fhe(value); + + Ok(()) + } } pub struct KeyshareFeature { @@ -88,7 +108,44 @@ impl E3Feature for KeyshareFeature { ); } - async fn hydrate(&self, _ctx: &mut E3RequestContext, _snapshot: &E3RequestContextSnapshot) {} + async fn hydrate( + &self, + ctx: &mut E3RequestContext, + snapshot: &E3RequestContextSnapshot, + ) -> Result<()> { + // No ID on the snapshot -> bail + let Some(id) = snapshot.keyshare.clone() else { + return Ok(()); + }; + + // No Snapshot returned from the store -> bail + let Some(snap) = ctx.store.at("").read(&id).await? else { + return Ok(()); + }; + + // Get deps + let Some(fhe) = ctx.fhe.clone() else { + return Ok(()); + }; + + // Construct from snapshot + let value = Keyshare::from_snapshot( + KeyshareParams { + fhe, + bus: self.bus.clone(), + store: ctx.store.at(&id), + address: self.address.clone(), + }, + snap, + ) + .await? + .start(); + + // send to context + ctx.set_keyshare(value); + + Ok(()) + } } pub struct PlaintextAggregatorFeature { @@ -138,7 +195,50 @@ impl E3Feature for PlaintextAggregatorFeature { .start(), ); } - async fn hydrate(&self, _ctx: &mut E3RequestContext, _snapshot: &E3RequestContextSnapshot) {} + + async fn hydrate( + &self, + ctx: &mut E3RequestContext, + snapshot: &E3RequestContextSnapshot, + ) -> Result<()> { + // No ID on the snapshot -> bail + let Some(id) = snapshot.plaintext.clone() else { + return Ok(()); + }; + + // No Snapshot returned from the store -> bail + let Some(snap) = ctx.store.at("").read(&id).await? else { + return Ok(()); + }; + + // Get deps + let Some(fhe) = ctx.fhe.clone() else { + return Ok(()); + }; + + let Some(meta) = ctx.meta.clone() else { + return Ok(()); + }; + + let value = PlaintextAggregator::from_snapshot( + PlaintextAggregatorParams { + fhe: fhe.clone(), + bus: self.bus.clone(), + store: ctx.get_store().at(&id), + sortition: self.sortition.clone(), + e3_id: ctx.e3_id.clone(), + src_chain_id: meta.src_chain_id, + }, + snap, + ) + .await? + .start(); + + // send to context + ctx.set_plaintext(value); + + Ok(()) + } } pub struct PublicKeyAggregatorFeature { @@ -188,5 +288,47 @@ impl E3Feature for PublicKeyAggregatorFeature { ); } - async fn hydrate(&self, _ctx: &mut E3RequestContext, _snapshot: &E3RequestContextSnapshot) {} + async fn hydrate( + &self, + ctx: &mut E3RequestContext, + snapshot: &E3RequestContextSnapshot, + ) -> Result<()> { + // No ID on the snapshot -> bail + let Some(id) = snapshot.publickey.clone() else { + return Ok(()); + }; + + // No Snapshot returned from the store -> bail + let Some(snap) = ctx.store.at("").read(&id).await? else { + return Ok(()); + }; + + // Get deps + let Some(fhe) = ctx.fhe.clone() else { + return Ok(()); + }; + + let Some(meta) = ctx.meta.clone() else { + return Ok(()); + }; + + let value = PublicKeyAggregator::from_snapshot( + PublicKeyAggregatorParams { + fhe: fhe.clone(), + bus: self.bus.clone(), + store: ctx.get_store().at(&id), + sortition: self.sortition.clone(), + e3_id: ctx.e3_id.clone(), + src_chain_id: meta.src_chain_id, + }, + snap, + ) + .await? + .start(); + + // send to context + ctx.set_publickey(value); + + Ok(()) + } } diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index b53db40f..7ca67949 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -29,7 +29,12 @@ use tokio::sync::Mutex; use tokio::{sync::mpsc::channel, time::sleep}; // Simulating a local node -async fn setup_local_ciphernode(bus: Addr, rng: SharedRng, logging: bool, addr: &str) { +async fn setup_local_ciphernode( + bus: Addr, + rng: SharedRng, + logging: bool, + addr: &str, +) -> Result<()> { // create data actor for saving data let data_actor = InMemDataStore::new(logging).start(); // TODO: Use a sled backed Data Actor let store = DataStore::from_in_mem(data_actor); @@ -49,9 +54,11 @@ async fn setup_local_ciphernode(bus: Addr, rng: SharedRng, logging: bo sortition.clone(), )) .add_feature(KeyshareFeature::create(bus.clone(), addr)) - .build(); + .build() + .await?; SimpleLogger::attach(addr, bus.clone()); + Ok(()) } fn generate_pk_share( @@ -75,9 +82,9 @@ async fn test_public_key_aggregation_and_decryption() -> Result<()> { .map(|_| Address::from_slice(&rand::thread_rng().gen::<[u8; 20]>()).to_string()) .collect(); - setup_local_ciphernode(bus.clone(), rng.clone(), true, ð_addrs[0]).await; - setup_local_ciphernode(bus.clone(), rng.clone(), true, ð_addrs[1]).await; - setup_local_ciphernode(bus.clone(), rng.clone(), true, ð_addrs[2]).await; + setup_local_ciphernode(bus.clone(), rng.clone(), true, ð_addrs[0]).await?; + setup_local_ciphernode(bus.clone(), rng.clone(), true, ð_addrs[1]).await?; + setup_local_ciphernode(bus.clone(), rng.clone(), true, ð_addrs[2]).await?; let e3_id = E3id::new("1234"); From 32ade4e18e9a4a751c1eb4984a043e7bc7292aa8 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 16 Oct 2024 00:37:04 +1100 Subject: [PATCH 08/49] Tidy up names --- packages/ciphernode/data/src/data_store.rs | 11 ++++++++++- .../ciphernode/router/src/committee_meta.rs | 18 +++++++++++++++--- .../ciphernode/router/src/e3_request_router.rs | 6 +++--- packages/ciphernode/router/src/hooks.rs | 18 ++++++++++-------- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index 72c83511..faff7eb3 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -66,7 +66,7 @@ pub trait Checkpoint: Snapshot { #[async_trait] pub trait FromSnapshotWithParams: Snapshot { - type Params:Send + 'static; + type Params: Send + 'static; /// Return an instance of the persistable object at the state given by the snapshot /// This method is async because there may be subobjects that require hydration from the store @@ -181,6 +181,15 @@ impl DataStore { self.set("", value) } + /// Read the value of the key starting at the root + pub async fn read_at(&self, key: K) -> Result> + where + K: IntoKey, + T: for<'de> Deserialize<'de>, + { + self.at("").read(key).await + } + // use this for testing pub fn from_in_mem(addr: Addr) -> Self { Self { diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index 6eee0497..4bc1cd0e 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -21,7 +21,7 @@ impl CommitteMetaFeature { #[async_trait] impl E3Feature for CommitteMetaFeature { - fn event(&self, ctx: &mut crate::E3RequestContext, event: &EnclaveEvent) { + fn on_event(&self, ctx: &mut crate::E3RequestContext, event: &EnclaveEvent) { let EnclaveEvent::E3Requested { data, .. } = event else { return; }; @@ -46,9 +46,21 @@ impl E3Feature for CommitteMetaFeature { async fn hydrate( &self, - _ctx: &mut E3RequestContext, - _snapshot: &E3RequestContextSnapshot, + ctx: &mut E3RequestContext, + snapshot: &E3RequestContextSnapshot, ) -> Result<()> { + // No ID on the snapshot -> bail + let Some(id) = snapshot.meta.clone() else { + return Ok(()); + }; + + // No Snapshot returned from the store -> bail + let Some(value) = ctx.store.read_at(&id).await? else { + return Ok(()); + }; + + ctx.set_meta(value); + Ok(()) } } diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index cc664461..e6a95a10 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -231,7 +231,7 @@ impl Checkpoint for E3RequestContext { #[async_trait] pub trait E3Feature: Send + Sync + 'static { - fn event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent); + fn on_event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent); async fn hydrate( &self, ctx: &mut E3RequestContext, @@ -311,7 +311,7 @@ impl Handler for E3RequestRouter { }); for feature in self.features.clone().iter() { - feature.event(context, &msg); + feature.on_event(context, &msg); } context.forward_message(&msg, &mut self.buffer); @@ -416,7 +416,7 @@ impl E3RequestRouterBuilder { } pub async fn build(self) -> Result> { - let snapshot: Option = self.store.at("").read("//router").await?; + let snapshot: Option = self.store.read_at("//router").await?; let params = E3RequestRouterParams { features: self.features.into(), bus: self.bus.clone(), diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index 795a8053..810d723b 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -13,6 +13,8 @@ use keyshare::{Keyshare, KeyshareParams}; use sortition::Sortition; use std::sync::Arc; +/// TODO: move these to each package with access on MyStruct::launcher() + pub struct FheFeature { rng: SharedRng, } @@ -25,7 +27,7 @@ impl FheFeature { #[async_trait] impl E3Feature for FheFeature { - fn event(&self, ctx: &mut crate::E3RequestContext, evt: &EnclaveEvent) { + fn on_event(&self, ctx: &mut crate::E3RequestContext, evt: &EnclaveEvent) { // Saving the fhe on Committee Requested let EnclaveEvent::E3Requested { data, .. } = evt else { return; @@ -56,7 +58,7 @@ impl E3Feature for FheFeature { }; // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.at("").read(&id).await? else { + let Some(snap) = ctx.store.read_at(&id).await? else { return Ok(()); }; @@ -83,7 +85,7 @@ impl KeyshareFeature { #[async_trait] impl E3Feature for KeyshareFeature { - fn event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { + fn on_event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { // Save Ciphernode on CiphernodeSelected let EnclaveEvent::CiphernodeSelected { data, .. } = evt else { return; @@ -119,7 +121,7 @@ impl E3Feature for KeyshareFeature { }; // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.at("").read(&id).await? else { + let Some(snap) = ctx.store.read_at(&id).await? else { return Ok(()); }; @@ -160,7 +162,7 @@ impl PlaintextAggregatorFeature { #[async_trait] impl E3Feature for PlaintextAggregatorFeature { - fn event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { + fn on_event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { // Save plaintext aggregator let EnclaveEvent::CiphertextOutputPublished { data, .. } = evt else { return; @@ -207,7 +209,7 @@ impl E3Feature for PlaintextAggregatorFeature { }; // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.at("").read(&id).await? else { + let Some(snap) = ctx.store.read_at(&id).await? else { return Ok(()); }; @@ -254,7 +256,7 @@ impl PublicKeyAggregatorFeature { #[async_trait] impl E3Feature for PublicKeyAggregatorFeature { - fn event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { + fn on_event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { // Saving the publickey aggregator with deps on E3Requested let EnclaveEvent::E3Requested { data, .. } = evt else { return; @@ -299,7 +301,7 @@ impl E3Feature for PublicKeyAggregatorFeature { }; // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.at("").read(&id).await? else { + let Some(snap) = ctx.store.read_at(&id).await? else { return Ok(()); }; From 61cf59ca574bfa692abd600ab217998ae96f52bb Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 16 Oct 2024 16:46:54 +1100 Subject: [PATCH 09/49] Add Hydration for Sortition --- packages/ciphernode/Cargo.lock | 3 ++ .../ciphernode/enclave_node/src/aggregator.rs | 2 +- .../ciphernode/enclave_node/src/ciphernode.rs | 2 +- packages/ciphernode/sortition/Cargo.toml | 11 +++-- .../ciphernode/sortition/src/sortition.rs | 47 +++++++++++++++++-- .../tests/test_aggregation_and_decryption.rs | 2 +- 6 files changed, 56 insertions(+), 11 deletions(-) diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index c2b05562..dd73bb34 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -5632,9 +5632,12 @@ dependencies = [ "actix", "alloy", "anyhow", + "async-trait", + "data", "enclave-core", "num", "rand", + "serde", "tracing", ] diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 1dd4cb35..9ef5f801 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -53,7 +53,7 @@ impl MainAggregator { )); let store = DataStore::from_in_mem(InMemDataStore::new(true).start()); - let sortition = Sortition::attach(bus.clone()); + let sortition = Sortition::attach(bus.clone(), store.clone()); let signer = pull_eth_signer_from_env("PRIVATE_KEY").await?; for chain in config .chains diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 94f2f92e..c3672d56 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -59,7 +59,7 @@ impl MainCiphernode { let bus = EventBus::new(true).start(); // TODO: switch to Sled actor let data = DataStore::from_in_mem(InMemDataStore::new(true).start()); - let sortition = Sortition::attach(bus.clone()); + let sortition = Sortition::attach(bus.clone(), data.clone()); let selector = CiphernodeSelector::attach(bus.clone(), sortition.clone(), &address.to_string()); diff --git a/packages/ciphernode/sortition/Cargo.toml b/packages/ciphernode/sortition/Cargo.toml index 46d9f893..383e1955 100644 --- a/packages/ciphernode/sortition/Cargo.toml +++ b/packages/ciphernode/sortition/Cargo.toml @@ -9,10 +9,13 @@ path = "src/lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -num = { workspace = true } -rand = { workspace = true } -alloy = { workspace = true, features = ["full"] } actix = { workspace = true } -enclave-core = { path = "../core" } +alloy = { workspace = true, features = ["full"] } anyhow = { workspace = true } +async-trait = { workspace = true } +data = { path = "../data" } +enclave-core = { path = "../core" } +num = { workspace = true } +rand = { workspace = true } +serde = { workspace = true } tracing = { workspace = true } diff --git a/packages/ciphernode/sortition/src/sortition.rs b/packages/ciphernode/sortition/src/sortition.rs index d3cdd08e..bd582537 100644 --- a/packages/ciphernode/sortition/src/sortition.rs +++ b/packages/ciphernode/sortition/src/sortition.rs @@ -2,6 +2,8 @@ use crate::DistanceSortition; use actix::prelude::*; use alloy::primitives::Address; use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use data::{Checkpoint, DataStore, FromSnapshotWithParams, Snapshot}; use enclave_core::{ BusError, CiphernodeAdded, CiphernodeRemoved, EnclaveErrorType, EnclaveEvent, EventBus, Seed, Subscribe, @@ -22,6 +24,7 @@ pub trait SortitionList { fn remove(&mut self, address: T); } +#[derive(Clone, serde::Serialize, serde::Deserialize)] pub struct SortitionModule { nodes: HashSet, } @@ -81,18 +84,29 @@ pub struct GetNodes; pub struct Sortition { list: SortitionModule, bus: Addr, + store: DataStore, +} + +pub struct SortitionParams { + pub bus: Addr, + pub store: DataStore, } impl Sortition { - pub fn new(bus: Addr) -> Self { + pub fn new(params: SortitionParams) -> Self { Self { list: SortitionModule::new(), - bus, + bus: params.bus, + store: params.store, } } - pub fn attach(bus: Addr) -> Addr { - let addr = Sortition::new(bus.clone()).start(); + pub fn attach(bus: Addr, store: DataStore) -> Addr { + let addr = Sortition::new(SortitionParams { + bus: bus.clone(), + store, + }) + .start(); bus.do_send(Subscribe::new("CiphernodeAdded", addr.clone().into())); addr } @@ -106,6 +120,31 @@ impl Actor for Sortition { type Context = actix::Context; } +impl Snapshot for Sortition { + type Snapshot = SortitionModule; + fn snapshot(&self) -> Self::Snapshot { + self.list.clone() + } +} + +#[async_trait] +impl FromSnapshotWithParams for Sortition { + type Params = SortitionParams; + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + Ok(Sortition { + bus: params.bus, + store: params.store, + list: snapshot, + }) + } +} + +impl Checkpoint for Sortition { + fn get_store(&self) -> DataStore { + self.store.clone() + } +} + impl Handler for Sortition { type Result = (); fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index c2c0bd88..6f82577c 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -40,7 +40,7 @@ async fn setup_local_ciphernode( let store = DataStore::from_in_mem(data_actor); // create ciphernode actor for managing ciphernode flow - let sortition = Sortition::attach(bus.clone()); + let sortition = Sortition::attach(bus.clone(), store.clone()); CiphernodeSelector::attach(bus.clone(), sortition.clone(), addr); E3RequestRouter::builder(bus.clone(), store) From a3a5c5f762324561e96fddf52073eee4dfe239f2 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 16 Oct 2024 18:03:00 +1100 Subject: [PATCH 10/49] Add docs --- packages/ciphernode/data/src/data_store.rs | 16 ++++++++++++++++ packages/ciphernode/sortition/src/sortition.rs | 3 +++ 2 files changed, 19 insertions(+) diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index faff7eb3..e771bfa1 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -4,49 +4,62 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use serde::{de::DeserializeOwned, Deserialize, Serialize}; + +/// This trait allows our keys to be responsive to multiple inputs pub trait IntoKey { fn into_key(self) -> Vec; } +/// Keys can be vectors of String impl IntoKey for Vec { fn into_key(self) -> Vec { self.join("/").into_bytes() } } +/// Keys can be vectors of &str impl<'a> IntoKey for Vec<&'a str> { fn into_key(self) -> Vec { self.join("/").into_bytes() } } +/// Keys can be String impl IntoKey for String { fn into_key(self) -> Vec { self.into_bytes() } } +/// Keys can be &String impl IntoKey for &String { fn into_key(self) -> Vec { self.as_bytes().to_vec() } } +/// Keys can be &str impl<'a> IntoKey for &'a str { fn into_key(self) -> Vec { self.as_bytes().to_vec() } } +/// Trait to add a prefix to a data storage object. This is used as a recursive trait for setting +/// scope on data objects which include the Get ad Insert commands for the data store pub trait WithPrefix: Sized { fn prefix(&self, prefix: &str) -> Self; fn at(&self, key: &str) -> Self; } + +/// This trait enables the self type to report their state snapshot pub trait Snapshot where Self: Sized, { + /// The state must be serializable so that it can be stored as a value + /// The Snapshot should represent all the dynamic data managed within the Actor or Object type Snapshot: Serialize + DeserializeOwned; /// Return a tuple with the first element being the id string of the object and the second @@ -54,6 +67,8 @@ where fn snapshot(&self) -> Self::Snapshot; } + +/// This trait enables the self type to checkpoint its state pub trait Checkpoint: Snapshot { /// Declare the DataStore instance available on the object fn get_store(&self) -> DataStore; @@ -64,6 +79,7 @@ pub trait Checkpoint: Snapshot { } } +/// Enable the self type to be reconstituted from the parameters coupled with the Snapshot #[async_trait] pub trait FromSnapshotWithParams: Snapshot { type Params: Send + 'static; diff --git a/packages/ciphernode/sortition/src/sortition.rs b/packages/ciphernode/sortition/src/sortition.rs index bd582537..980b3ccd 100644 --- a/packages/ciphernode/sortition/src/sortition.rs +++ b/packages/ciphernode/sortition/src/sortition.rs @@ -190,3 +190,6 @@ impl Handler for Sortition { self.get_nodes() } } + + + From 915d31b14dc80d74067efbf9496aef137be40d6d Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 16 Oct 2024 18:21:56 +1100 Subject: [PATCH 11/49] Update docs --- packages/ciphernode/aggregator/src/lib.rs | 9 ++++++--- packages/ciphernode/data/src/data_store.rs | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/ciphernode/aggregator/src/lib.rs b/packages/ciphernode/aggregator/src/lib.rs index db42567c..fef3eb91 100644 --- a/packages/ciphernode/aggregator/src/lib.rs +++ b/packages/ciphernode/aggregator/src/lib.rs @@ -1,5 +1,8 @@ mod plaintext_aggregator; mod publickey_aggregator; - -pub use plaintext_aggregator::*; -pub use publickey_aggregator::*; +pub use plaintext_aggregator::{ + PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorState, +}; +pub use publickey_aggregator::{ + PublicKeyAggregator, PublicKeyAggregatorParams, PublicKeyAggregatorState, +}; diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index e771bfa1..78d72281 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -58,12 +58,12 @@ pub trait Snapshot where Self: Sized, { - /// The state must be serializable so that it can be stored as a value /// The Snapshot should represent all the dynamic data managed within the Actor or Object + /// + /// The state must be serializable so that it can be stored as a value type Snapshot: Serialize + DeserializeOwned; - /// Return a tuple with the first element being the id string of the object and the second - /// being a representation of the object's state that is easily serialized by the data store + /// Return the Snapshot object for the implementor fn snapshot(&self) -> Self::Snapshot; } From e17143f2cc3c88f374e5bde9eb1a826cb3b92413 Mon Sep 17 00:00:00 2001 From: ryardley Date: Wed, 16 Oct 2024 18:23:21 +1100 Subject: [PATCH 12/49] Formatting --- packages/ciphernode/data/src/data_store.rs | 3 --- packages/ciphernode/sortition/src/sortition.rs | 3 --- 2 files changed, 6 deletions(-) diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index 78d72281..8cda94ab 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -4,7 +4,6 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use serde::{de::DeserializeOwned, Deserialize, Serialize}; - /// This trait allows our keys to be responsive to multiple inputs pub trait IntoKey { fn into_key(self) -> Vec; @@ -52,7 +51,6 @@ pub trait WithPrefix: Sized { fn at(&self, key: &str) -> Self; } - /// This trait enables the self type to report their state snapshot pub trait Snapshot where @@ -67,7 +65,6 @@ where fn snapshot(&self) -> Self::Snapshot; } - /// This trait enables the self type to checkpoint its state pub trait Checkpoint: Snapshot { /// Declare the DataStore instance available on the object diff --git a/packages/ciphernode/sortition/src/sortition.rs b/packages/ciphernode/sortition/src/sortition.rs index 980b3ccd..bd582537 100644 --- a/packages/ciphernode/sortition/src/sortition.rs +++ b/packages/ciphernode/sortition/src/sortition.rs @@ -190,6 +190,3 @@ impl Handler for Sortition { self.get_nodes() } } - - - From 0905d1385406e793b756a8d992aae237ed66ff0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Wed, 16 Oct 2024 20:48:51 +1100 Subject: [PATCH 13/49] Update packages/ciphernode/router/src/committee_meta.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- packages/ciphernode/router/src/committee_meta.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index 4bc1cd0e..2cb281a3 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -11,16 +11,16 @@ pub struct CommitteeMeta { pub src_chain_id: u64, } -pub struct CommitteMetaFeature; +pub struct CommitteeMetaFeature; -impl CommitteMetaFeature { +impl CommitteeMetaFeature { pub fn create() -> Box { Box::new(Self {}) } } #[async_trait] -impl E3Feature for CommitteMetaFeature { +impl E3Feature for CommitteeMetaFeature { fn on_event(&self, ctx: &mut crate::E3RequestContext, event: &EnclaveEvent) { let EnclaveEvent::E3Requested { data, .. } = event else { return; From ff8e5f5064f93803acd16dfcfc346fb72c93c669 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Fri, 18 Oct 2024 12:05:30 +1100 Subject: [PATCH 14/49] Simplify DataStore Api --- packages/ciphernode/Cargo.lock | 1 + packages/ciphernode/data/Cargo.toml | 3 + packages/ciphernode/data/src/data_store.rs | 219 ++++-------------- packages/ciphernode/data/src/in_mem.rs | 12 +- packages/ciphernode/data/src/into_key.rs | 53 +++++ packages/ciphernode/data/src/lib.rs | 5 + packages/ciphernode/data/src/snapshot.rs | 46 ++++ .../ciphernode/enclave_node/src/aggregator.rs | 4 +- .../ciphernode/enclave_node/src/ciphernode.rs | 4 +- .../ciphernode/keyshare/src/repository.rs | 4 + .../ciphernode/router/src/committee_meta.rs | 5 +- .../router/src/e3_request_router.rs | 17 +- packages/ciphernode/router/src/hooks.rs | 24 +- .../tests/test_aggregation_and_decryption.rs | 4 +- 14 files changed, 193 insertions(+), 208 deletions(-) create mode 100644 packages/ciphernode/data/src/into_key.rs create mode 100644 packages/ciphernode/data/src/snapshot.rs create mode 100644 packages/ciphernode/keyshare/src/repository.rs diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index dd73bb34..2a795236 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -1845,6 +1845,7 @@ name = "data" version = "0.1.0" dependencies = [ "actix", + "actix-rt", "anyhow", "async-trait", "bincode", diff --git a/packages/ciphernode/data/Cargo.toml b/packages/ciphernode/data/Cargo.toml index dbac17ad..387d1ffd 100644 --- a/packages/ciphernode/data/Cargo.toml +++ b/packages/ciphernode/data/Cargo.toml @@ -11,3 +11,6 @@ anyhow = { workspace = true } serde = { workspace = true } bincode = { workspace = true } async-trait = { workspace = true } + +[dev-dependencies] +actix-rt = { workspace = true } diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index 8cda94ab..fdf0a5cc 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -1,111 +1,7 @@ -use crate::InMemDataStore; +use crate::{InMemStore, IntoKey}; use actix::{Addr, Message, Recipient}; -use anyhow::{anyhow, Result}; -use async_trait::async_trait; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; - -/// This trait allows our keys to be responsive to multiple inputs -pub trait IntoKey { - fn into_key(self) -> Vec; -} - -/// Keys can be vectors of String -impl IntoKey for Vec { - fn into_key(self) -> Vec { - self.join("/").into_bytes() - } -} - -/// Keys can be vectors of &str -impl<'a> IntoKey for Vec<&'a str> { - fn into_key(self) -> Vec { - self.join("/").into_bytes() - } -} - -/// Keys can be String -impl IntoKey for String { - fn into_key(self) -> Vec { - self.into_bytes() - } -} - -/// Keys can be &String -impl IntoKey for &String { - fn into_key(self) -> Vec { - self.as_bytes().to_vec() - } -} - -/// Keys can be &str -impl<'a> IntoKey for &'a str { - fn into_key(self) -> Vec { - self.as_bytes().to_vec() - } -} - -/// Trait to add a prefix to a data storage object. This is used as a recursive trait for setting -/// scope on data objects which include the Get ad Insert commands for the data store -pub trait WithPrefix: Sized { - fn prefix(&self, prefix: &str) -> Self; - fn at(&self, key: &str) -> Self; -} - -/// This trait enables the self type to report their state snapshot -pub trait Snapshot -where - Self: Sized, -{ - /// The Snapshot should represent all the dynamic data managed within the Actor or Object - /// - /// The state must be serializable so that it can be stored as a value - type Snapshot: Serialize + DeserializeOwned; - - /// Return the Snapshot object for the implementor - fn snapshot(&self) -> Self::Snapshot; -} - -/// This trait enables the self type to checkpoint its state -pub trait Checkpoint: Snapshot { - /// Declare the DataStore instance available on the object - fn get_store(&self) -> DataStore; - - /// Write the current snapshot to the DataStore provided by `get_store()` at the object's id returned by `get_id()` - fn checkpoint(&self) { - self.get_store().write(self.snapshot()); - } -} - -/// Enable the self type to be reconstituted from the parameters coupled with the Snapshot -#[async_trait] -pub trait FromSnapshotWithParams: Snapshot { - type Params: Send + 'static; - - /// Return an instance of the persistable object at the state given by the snapshot - /// This method is async because there may be subobjects that require hydration from the store - async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result; -} - -#[async_trait] -pub trait FromSnapshot: Snapshot { - /// Return an instance of the persistable object at the state given by the snapshot - /// This method is async because there may be subobjects that require hydration from the store - async fn from_snapshot(snapshot: Self::Snapshot) -> Result; -} - -impl WithPrefix for Vec { - fn prefix(&self, prefix: &str) -> Self { - let Ok(encoded) = String::from_utf8(self.clone()) else { - // If this is not encoded as utf8 do nothing - return self.clone(); - }; - vec![prefix.to_string(), encoded].join("/").into_bytes() - } - - fn at(&self, key: &str) -> Self { - key.to_string().into_bytes() - } -} +use anyhow::Result; +use serde::{Deserialize, Serialize}; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "()")] @@ -124,16 +20,6 @@ impl Insert { } } -impl WithPrefix for Insert { - fn prefix(&self, prefix: &str) -> Self { - Insert(self.0.prefix(prefix), self.1.clone()) - } - - fn at(&self, key: &str) -> Self { - Insert(self.0.at(key), self.1.clone()) - } -} - #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "Option>")] pub struct Get(pub Vec); @@ -147,102 +33,87 @@ impl Get { } } -impl WithPrefix for Get { - fn prefix(&self, prefix: &str) -> Self { - Get(self.0.prefix(prefix)) - } - fn at(&self, key: &str) -> Self { - Get(self.0.at(key)) - } -} - #[derive(Clone)] pub struct DataStore { - prefix: Option, + scope: Vec, get: Recipient, insert: Recipient, } impl DataStore { - pub async fn read(&self, key: K) -> Result> + /// Read data at the scope location + pub async fn read(&self) -> Result> where - K: IntoKey, T: for<'de> Deserialize<'de>, { - let msg = Get::new(key); - let msg = self.prefix.as_ref().map_or(msg.clone(), |p| msg.prefix(p)); - let maybe_bytes = self.get.send(msg).await?; - let Some(bytes) = maybe_bytes else { + let Some(bytes) = self.get.send(Get::new(&self.scope)).await? else { return Ok(None); }; Ok(Some(bincode::deserialize(&bytes)?)) } - /// Writes anything serializable to the KV actor as a stream of bytes - pub fn set(&self, key: K, value: V) { + /// Writes data to the scope location + pub fn write(&self, value: T) { let Ok(serialized) = bincode::serialize(&value) else { return; }; - let msg = Insert::new(key, serialized); - let msg = self.prefix.as_ref().map_or(msg.clone(), |p| msg.prefix(p)); + let msg = Insert::new(&self.scope, serialized); self.insert.do_send(msg) } - /// Writes to whatever the prefix is set to on the datastore - pub fn write(&self, value: V) { - self.set("", value) - } - - /// Read the value of the key starting at the root - pub async fn read_at(&self, key: K) -> Result> - where - K: IntoKey, - T: for<'de> Deserialize<'de>, - { - self.at("").read(key).await - } - - // use this for testing - pub fn from_in_mem(addr: Addr) -> Self { + /// Construct a data store from an InMemStore actor + pub fn from_in_mem(addr: Addr) -> Self { Self { get: addr.clone().recipient(), insert: addr.clone().recipient(), - prefix: None, + scope: vec![], } } - pub fn ensure_root_id(str: &str) -> Result<()> { - if !str.starts_with("/") { - return Err(anyhow!("string doesnt start with slash.")); - } - Ok(()) + /// Get the scope as a string + pub fn get_scope(&self) -> Result { + Ok(String::from_utf8(self.scope.clone())?) } - // // use this for production - // pub fn from_sled(&data_addr: Addr) -> Self { - // let d = data_addr.clone(); - // Self(d.recipient(),d.recipient()) - // } -} - -impl WithPrefix for DataStore { - fn prefix(&self, prefix: &str) -> Self { + /// Changes the scope for the data store. + /// Note that if the scope does not start with a slash one is appended. + /// ``` + /// use data::DataStore; + /// use data::InMemStore; + /// use actix::Actor; + /// use anyhow::Result; + /// + /// #[actix_rt::main] + /// async fn main() -> Result<()>{ + /// let addr = InMemStore::new(false).start(); + /// let store = DataStore::from_in_mem(addr); + /// assert_eq!(store.base("//foo") + /// .scope("bar") + /// .scope("/baz") + /// .get_scope()?, "//foo/bar/baz"); + /// Ok(()) + /// } + /// ``` + pub fn scope(&self, key: K) -> Self { + let mut scope = self.scope.clone(); + let encoded_key = key.into_key(); + if !encoded_key.starts_with(&[b'/']) { + scope.extend("/".into_key()); + } + scope.extend(encoded_key); Self { get: self.get.clone(), insert: self.insert.clone(), - prefix: self.prefix.clone().map_or_else( - || Some(prefix.to_string()), - |p| Some(vec![prefix.to_string(), p].join("/")), - ), + scope, } } - fn at(&self, key: &str) -> Self { + pub fn base(&self, key: K) -> Self { Self { get: self.get.clone(), insert: self.insert.clone(), - prefix: Some(key.to_string()), + scope: key.into_key(), } } } diff --git a/packages/ciphernode/data/src/in_mem.rs b/packages/ciphernode/data/src/in_mem.rs index 1aa4993e..6821de72 100644 --- a/packages/ciphernode/data/src/in_mem.rs +++ b/packages/ciphernode/data/src/in_mem.rs @@ -12,17 +12,17 @@ pub enum DataOp { Insert(Insert), } -pub struct InMemDataStore { +pub struct InMemStore { db: BTreeMap, Vec>, log: Vec, capture: bool, } -impl Actor for InMemDataStore { +impl Actor for InMemStore { type Context = Context; } -impl InMemDataStore { +impl InMemStore { pub fn new(capture: bool) -> Self { Self { db: BTreeMap::new(), @@ -32,7 +32,7 @@ impl InMemDataStore { } } -impl Handler for InMemDataStore { +impl Handler for InMemStore { type Result = (); fn handle(&mut self, event: Insert, _: &mut Self::Context) { // insert data into sled @@ -44,7 +44,7 @@ impl Handler for InMemDataStore { } } -impl Handler for InMemDataStore { +impl Handler for InMemStore { type Result = Option>; fn handle(&mut self, event: Get, _: &mut Self::Context) -> Option> { let key = event.key(); @@ -52,7 +52,7 @@ impl Handler for InMemDataStore { } } -impl Handler for InMemDataStore { +impl Handler for InMemStore { type Result = Vec; fn handle(&mut self, _: GetLog, _: &mut Self::Context) -> Vec { self.log.clone() diff --git a/packages/ciphernode/data/src/into_key.rs b/packages/ciphernode/data/src/into_key.rs new file mode 100644 index 00000000..5da1c550 --- /dev/null +++ b/packages/ciphernode/data/src/into_key.rs @@ -0,0 +1,53 @@ +/// This trait allows our keys to be responsive to multiple inputs +pub trait IntoKey { + fn into_key(self) -> Vec; +} + +/// Keys can be vectors of String +impl IntoKey for Vec { + fn into_key(self) -> Vec { + self.clone() + } +} + +/// Keys can be vectors of String +impl IntoKey for &Vec { + fn into_key(self) -> Vec { + self.clone() + } +} + +/// Keys can be vectors of String +impl IntoKey for Vec { + fn into_key(self) -> Vec { + self.join("/").into_bytes() + } +} + +/// Keys can be vectors of &str +impl<'a> IntoKey for Vec<&'a str> { + fn into_key(self) -> Vec { + self.join("/").into_bytes() + } +} + +/// Keys can be String +impl IntoKey for String { + fn into_key(self) -> Vec { + self.into_bytes() + } +} + +/// Keys can be &String +impl IntoKey for &String { + fn into_key(self) -> Vec { + self.as_bytes().to_vec() + } +} + +/// Keys can be &str +impl<'a> IntoKey for &'a str { + fn into_key(self) -> Vec { + self.as_bytes().to_vec() + } +} diff --git a/packages/ciphernode/data/src/lib.rs b/packages/ciphernode/data/src/lib.rs index a439fca7..285a095a 100644 --- a/packages/ciphernode/data/src/lib.rs +++ b/packages/ciphernode/data/src/lib.rs @@ -1,4 +1,9 @@ mod data_store; mod in_mem; +mod into_key; +mod snapshot; + pub use data_store::*; pub use in_mem::*; +pub use into_key::IntoKey; +pub use snapshot::*; diff --git a/packages/ciphernode/data/src/snapshot.rs b/packages/ciphernode/data/src/snapshot.rs new file mode 100644 index 00000000..7204652a --- /dev/null +++ b/packages/ciphernode/data/src/snapshot.rs @@ -0,0 +1,46 @@ +use crate::DataStore; +use anyhow::Result; +use async_trait::async_trait; +use serde::{de::DeserializeOwned, Serialize}; + +/// This trait enables the self type to report their state snapshot +pub trait Snapshot +where + Self: Sized, +{ + /// The Snapshot should represent all the dynamic data managed within the Actor or Object + /// + /// The state must be serializable so that it can be stored as a value + type Snapshot: Serialize + DeserializeOwned; + + /// Return the Snapshot object for the implementor + fn snapshot(&self) -> Self::Snapshot; +} + +/// This trait enables the self type to checkpoint its state +pub trait Checkpoint: Snapshot { + /// Declare the DataStore instance available on the object + fn get_store(&self) -> DataStore; + + /// Write the current snapshot to the DataStore provided by `get_store()` at the object's id returned by `get_id()` + fn checkpoint(&self) { + self.get_store().write(self.snapshot()); + } +} + +/// Enable the self type to be reconstituted from the parameters coupled with the Snapshot +#[async_trait] +pub trait FromSnapshotWithParams: Snapshot { + type Params: Send + 'static; + + /// Return an instance of the persistable object at the state given by the snapshot + /// This method is async because there may be subobjects that require hydration from the store + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result; +} + +#[async_trait] +pub trait FromSnapshot: Snapshot { + /// Return an instance of the persistable object at the state given by the snapshot + /// This method is async because there may be subobjects that require hydration from the store + async fn from_snapshot(snapshot: Self::Snapshot) -> Result; +} diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 9ef5f801..63bd73e3 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -1,6 +1,6 @@ use actix::{Actor, Addr, Context}; use anyhow::Result; -use data::{DataStore, InMemDataStore}; +use data::{DataStore, InMemStore}; use enclave_core::EventBus; use evm::{ helpers::pull_eth_signer_from_env, CiphernodeRegistrySol, EnclaveSol, RegistryFilterSol, @@ -52,7 +52,7 @@ impl MainAggregator { rand_chacha::ChaCha20Rng::from_rng(OsRng).expect("Failed to create RNG"), )); - let store = DataStore::from_in_mem(InMemDataStore::new(true).start()); + let store = DataStore::from_in_mem(InMemStore::new(true).start()); let sortition = Sortition::attach(bus.clone(), store.clone()); let signer = pull_eth_signer_from_env("PRIVATE_KEY").await?; for chain in config diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index c3672d56..98040d07 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -1,7 +1,7 @@ use actix::{Actor, Addr, Context}; use alloy::primitives::Address; use anyhow::Result; -use data::{DataStore, InMemDataStore}; +use data::{DataStore, InMemStore}; use enclave_core::EventBus; use evm::{CiphernodeRegistrySol, EnclaveSolReader}; use logger::SimpleLogger; @@ -58,7 +58,7 @@ impl MainCiphernode { )); let bus = EventBus::new(true).start(); // TODO: switch to Sled actor - let data = DataStore::from_in_mem(InMemDataStore::new(true).start()); + let data = DataStore::from_in_mem(InMemStore::new(true).start()); let sortition = Sortition::attach(bus.clone(), data.clone()); let selector = CiphernodeSelector::attach(bus.clone(), sortition.clone(), &address.to_string()); diff --git a/packages/ciphernode/keyshare/src/repository.rs b/packages/ciphernode/keyshare/src/repository.rs new file mode 100644 index 00000000..ed8b85f1 --- /dev/null +++ b/packages/ciphernode/keyshare/src/repository.rs @@ -0,0 +1,4 @@ +use data::{Repository,DataStore}; +use crate::keyshare::KeyshareState; + + diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index 2cb281a3..ab210143 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -1,7 +1,6 @@ use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot}; use anyhow::*; use async_trait::async_trait; -use data::WithPrefix; use enclave_core::{E3Requested, EnclaveEvent, Seed}; #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] @@ -40,7 +39,7 @@ impl E3Feature for CommitteeMetaFeature { seed, src_chain_id, }; - ctx.get_store().at(&meta_id).write(meta.clone()); + ctx.get_store().scope(&meta_id).write(meta.clone()); let _ = ctx.set_meta(meta); } @@ -55,7 +54,7 @@ impl E3Feature for CommitteeMetaFeature { }; // No Snapshot returned from the store -> bail - let Some(value) = ctx.store.read_at(&id).await? else { + let Some(value) = ctx.store.scope(&id).read().await? else { return Ok(()); }; diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index e6a95a10..df2996e8 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -1,4 +1,4 @@ -use crate::CommitteMetaFeature; +use crate::CommitteeMetaFeature; use super::CommitteeMeta; use actix::{Actor, Addr, Context, Handler, Recipient}; @@ -10,7 +10,6 @@ use data::Checkpoint; use data::DataStore; use data::FromSnapshotWithParams; use data::Snapshot; -use data::WithPrefix; use enclave_core::E3RequestComplete; use enclave_core::{E3id, EnclaveEvent, EventBus, Subscribe}; use fhe::Fhe; @@ -18,7 +17,6 @@ use keyshare::Keyshare; use serde::Deserialize; use serde::Serialize; use std::collections::HashSet; -use std::pin::Pin; use std::{collections::HashMap, sync::Arc}; /// Helper class to buffer events for downstream instances incase events arrive in the wrong order @@ -269,7 +267,7 @@ impl E3RequestRouter { }; // Everything needs the committe meta factory so adding it here by default - builder.add_feature(CommitteMetaFeature::create()) + builder.add_feature(CommitteeMetaFeature::create()) } pub fn from_params(params: E3RequestRouterParams) -> Self { @@ -305,7 +303,7 @@ impl Handler for E3RequestRouter { let context = self.contexts.entry(e3_id.clone()).or_insert_with(|| { E3RequestContext::from_params(E3RequestContextParams { e3_id: e3_id.clone(), - store: self.store.at(&format!("//context/{e3_id}")), + store: self.store.scope(&format!("//context/{e3_id}")), features: self.features.clone(), }) }); @@ -373,7 +371,12 @@ impl FromSnapshotWithParams for E3RequestRouter { async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { let mut contexts = HashMap::new(); for e3_id in snapshot.contexts { - let Some(ctx_snapshot) = params.store.read(format!("//context/{e3_id}")).await? else { + let Some(ctx_snapshot) = params + .store + .base(format!("//context/{e3_id}")) + .read() + .await? + else { continue; }; @@ -416,7 +419,7 @@ impl E3RequestRouterBuilder { } pub async fn build(self) -> Result> { - let snapshot: Option = self.store.read_at("//router").await?; + let snapshot: Option = self.store.base("//router").read().await?; let params = E3RequestRouterParams { features: self.features.into(), bus: self.bus.clone(), diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index 17f988a7..92dde57f 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -6,7 +6,7 @@ use aggregator::{ }; use anyhow::{anyhow, Result}; use async_trait::async_trait; -use data::{FromSnapshotWithParams, Snapshot, WithPrefix}; +use data::{FromSnapshotWithParams, Snapshot}; use enclave_core::{BusError, E3Requested, EnclaveErrorType, EnclaveEvent, EventBus}; use fhe::{Fhe, SharedRng}; use keyshare::{Keyshare, KeyshareParams}; @@ -42,7 +42,7 @@ impl E3Feature for FheFeature { // FHE doesn't implement Checkpoint so we are going to store it manually let fhe_id = format!("//fhe/{e3_id}"); let fhe = Arc::new(Fhe::from_encoded(¶ms, seed, self.rng.clone()).unwrap()); - ctx.get_store().at(&fhe_id).write(fhe.snapshot()); + ctx.get_store().scope(&fhe_id).write(fhe.snapshot()); let _ = ctx.set_fhe(fhe); } @@ -57,7 +57,7 @@ impl E3Feature for FheFeature { }; // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.read_at(&id).await? else { + let Some(snap) = ctx.store.scope(&id).read().await? else { return Ok(()); }; @@ -102,7 +102,7 @@ impl E3Feature for KeyshareFeature { let _ = ctx.set_keyshare( Keyshare::new(KeyshareParams { bus: self.bus.clone(), - store: ctx.get_store().at(&ks_id), + store: ctx.get_store().scope(&ks_id), fhe: fhe.clone(), address: self.address.clone(), }) @@ -121,7 +121,7 @@ impl E3Feature for KeyshareFeature { }; // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.read_at(&id).await? else { + let Some(snap) = ctx.store.scope(&id).read().await? else { return Ok(()); }; @@ -135,7 +135,7 @@ impl E3Feature for KeyshareFeature { KeyshareParams { fhe, bus: self.bus.clone(), - store: ctx.store.at(&id), + store: ctx.store.scope(&id), address: self.address.clone(), }, snap, @@ -187,7 +187,7 @@ impl E3Feature for PlaintextAggregatorFeature { PlaintextAggregatorParams { fhe: fhe.clone(), bus: self.bus.clone(), - store: ctx.get_store().at(id), + store: ctx.get_store().scope(id), sortition: self.sortition.clone(), e3_id, src_chain_id: meta.src_chain_id, @@ -213,7 +213,7 @@ impl E3Feature for PlaintextAggregatorFeature { }; // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.read_at(&id).await? else { + let Some(snap) = ctx.store.scope(&id).read().await? else { return Ok(()); }; @@ -230,7 +230,7 @@ impl E3Feature for PlaintextAggregatorFeature { PlaintextAggregatorParams { fhe: fhe.clone(), bus: self.bus.clone(), - store: ctx.get_store().at(&id), + store: ctx.get_store().scope(&id), sortition: self.sortition.clone(), e3_id: ctx.e3_id.clone(), src_chain_id: meta.src_chain_id, @@ -283,7 +283,7 @@ impl E3Feature for PublicKeyAggregatorFeature { PublicKeyAggregatorParams { fhe: fhe.clone(), bus: self.bus.clone(), - store: ctx.get_store().at(id), + store: ctx.get_store().scope(id), sortition: self.sortition.clone(), e3_id, src_chain_id: meta.src_chain_id, @@ -305,7 +305,7 @@ impl E3Feature for PublicKeyAggregatorFeature { }; // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.read_at(&id).await? else { + let Some(snap) = ctx.store.scope(&id).read().await? else { return Ok(()); }; @@ -322,7 +322,7 @@ impl E3Feature for PublicKeyAggregatorFeature { PublicKeyAggregatorParams { fhe: fhe.clone(), bus: self.bus.clone(), - store: ctx.get_store().at(&id), + store: ctx.get_store().scope(&id), sortition: self.sortition.clone(), e3_id: ctx.e3_id.clone(), src_chain_id: meta.src_chain_id, diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 6f82577c..88493815 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -1,4 +1,4 @@ -use data::{DataStore, InMemDataStore}; +use data::{DataStore, InMemStore}; use enclave_core::{ CiphernodeAdded, CiphernodeSelected, CiphertextOutputPublished, DecryptionshareCreated, E3RequestComplete, E3Requested, E3id, EnclaveEvent, EventBus, GetHistory, KeyshareCreated, @@ -36,7 +36,7 @@ async fn setup_local_ciphernode( addr: &str, ) -> Result<()> { // create data actor for saving data - let data_actor = InMemDataStore::new(logging).start(); // TODO: Use a sled backed Data Actor + let data_actor = InMemStore::new(logging).start(); // TODO: Use a sled backed Data Actor let store = DataStore::from_in_mem(data_actor); // create ciphernode actor for managing ciphernode flow From c8f31ea8f824f8551ce8c52394f7b85b689b8e18 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Fri, 18 Oct 2024 14:29:59 +1100 Subject: [PATCH 15/49] Apply first draft of repositories to data storage --- packages/ciphernode/aggregator/src/lib.rs | 4 + .../aggregator/src/plaintext_aggregator.rs | 11 +- .../aggregator/src/plaintext_repository.rs | 29 +++ .../aggregator/src/publickey_aggregator.rs | 11 +- .../aggregator/src/publickey_repository.rs | 29 +++ packages/ciphernode/data/src/data_store.rs | 1 + packages/ciphernode/data/src/lib.rs | 2 + packages/ciphernode/data/src/repository.rs | 18 ++ packages/ciphernode/data/src/snapshot.rs | 7 +- .../ciphernode/enclave_node/src/aggregator.rs | 4 +- .../ciphernode/enclave_node/src/ciphernode.rs | 4 +- packages/ciphernode/fhe/src/lib.rs | 2 + packages/ciphernode/fhe/src/repository.rs | 30 +++ packages/ciphernode/keyshare/src/keyshare.rs | 11 +- packages/ciphernode/keyshare/src/lib.rs | 2 + .../ciphernode/keyshare/src/repository.rs | 27 ++- .../ciphernode/router/src/committee_meta.rs | 11 +- .../router/src/committee_repository.rs | 30 +++ packages/ciphernode/router/src/context.rs | 179 ++++++++++++++ .../router/src/context_repository.rs | 30 +++ .../router/src/e3_request_router.rs | 229 ++---------------- packages/ciphernode/router/src/hooks.rs | 55 ++--- packages/ciphernode/router/src/lib.rs | 8 + .../router/src/router_repository.rs | 29 +++ packages/ciphernode/sortition/src/lib.rs | 2 + .../ciphernode/sortition/src/repository.rs | 37 +++ .../ciphernode/sortition/src/sortition.rs | 13 +- 27 files changed, 550 insertions(+), 265 deletions(-) create mode 100644 packages/ciphernode/aggregator/src/plaintext_repository.rs create mode 100644 packages/ciphernode/aggregator/src/publickey_repository.rs create mode 100644 packages/ciphernode/data/src/repository.rs create mode 100644 packages/ciphernode/fhe/src/repository.rs create mode 100644 packages/ciphernode/router/src/committee_repository.rs create mode 100644 packages/ciphernode/router/src/context.rs create mode 100644 packages/ciphernode/router/src/context_repository.rs create mode 100644 packages/ciphernode/router/src/router_repository.rs create mode 100644 packages/ciphernode/sortition/src/repository.rs diff --git a/packages/ciphernode/aggregator/src/lib.rs b/packages/ciphernode/aggregator/src/lib.rs index fef3eb91..59e3bbf6 100644 --- a/packages/ciphernode/aggregator/src/lib.rs +++ b/packages/ciphernode/aggregator/src/lib.rs @@ -1,8 +1,12 @@ mod plaintext_aggregator; +mod plaintext_repository; mod publickey_aggregator; +mod publickey_repository; pub use plaintext_aggregator::{ PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorState, }; +pub use plaintext_repository::*; pub use publickey_aggregator::{ PublicKeyAggregator, PublicKeyAggregatorParams, PublicKeyAggregatorState, }; +pub use publickey_repository::*; diff --git a/packages/ciphernode/aggregator/src/plaintext_aggregator.rs b/packages/ciphernode/aggregator/src/plaintext_aggregator.rs index 16fcfc24..fd60bead 100644 --- a/packages/ciphernode/aggregator/src/plaintext_aggregator.rs +++ b/packages/ciphernode/aggregator/src/plaintext_aggregator.rs @@ -1,7 +1,7 @@ use actix::prelude::*; use anyhow::Result; use async_trait::async_trait; -use data::{Checkpoint, DataStore, FromSnapshotWithParams, Snapshot}; +use data::{Checkpoint, FromSnapshotWithParams, Snapshot}; use enclave_core::{ DecryptionshareCreated, Die, E3id, EnclaveEvent, EventBus, OrderedSet, PlaintextAggregated, Seed, @@ -11,6 +11,8 @@ use sortition::{GetHasNode, Sortition}; use std::sync::Arc; use tracing::error; +use crate::plaintext_repository::PlaintextAggregatorRepository; + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum PlaintextAggregatorState { Collecting { @@ -50,7 +52,7 @@ struct ComputeAggregate { pub struct PlaintextAggregator { fhe: Arc, bus: Addr, - store: DataStore, + store: PlaintextAggregatorRepository, sortition: Addr, e3_id: E3id, state: PlaintextAggregatorState, @@ -60,7 +62,7 @@ pub struct PlaintextAggregator { pub struct PlaintextAggregatorParams { pub fhe: Arc, pub bus: Addr, - pub store: DataStore, + pub store: PlaintextAggregatorRepository, pub sortition: Addr, pub e3_id: E3id, pub src_chain_id: u64, @@ -236,7 +238,8 @@ impl FromSnapshotWithParams for PlaintextAggregator { } impl Checkpoint for PlaintextAggregator { - fn get_store(&self) -> data::DataStore { + type Repository = PlaintextAggregatorRepository; + fn get_store(&self) -> PlaintextAggregatorRepository { self.store.clone() } } diff --git a/packages/ciphernode/aggregator/src/plaintext_repository.rs b/packages/ciphernode/aggregator/src/plaintext_repository.rs new file mode 100644 index 00000000..4c19aeb1 --- /dev/null +++ b/packages/ciphernode/aggregator/src/plaintext_repository.rs @@ -0,0 +1,29 @@ +use crate::PlaintextAggregatorState; +use async_trait::async_trait; +use data::{DataStore, Repository}; +use enclave_core::E3id; + +#[derive(Clone)] +pub struct PlaintextAggregatorRepository { + store: DataStore, +} + +#[async_trait] +impl Repository for PlaintextAggregatorRepository { + type State = PlaintextAggregatorState; + fn store(&self) -> DataStore { + self.store.clone() + } +} + +pub trait PlaintextAggregatorRepositoryFactory { + fn plaintext(&self, e3_id: &E3id) -> PlaintextAggregatorRepository; +} + +impl PlaintextAggregatorRepositoryFactory for DataStore { + fn plaintext(&self, e3_id: &E3id) -> PlaintextAggregatorRepository { + PlaintextAggregatorRepository { + store: self.scope(format!("//plaintext/{e3_id}")), + } + } +} diff --git a/packages/ciphernode/aggregator/src/publickey_aggregator.rs b/packages/ciphernode/aggregator/src/publickey_aggregator.rs index 76774648..85a0dea4 100644 --- a/packages/ciphernode/aggregator/src/publickey_aggregator.rs +++ b/packages/ciphernode/aggregator/src/publickey_aggregator.rs @@ -1,7 +1,7 @@ use actix::prelude::*; use anyhow::Result; use async_trait::async_trait; -use data::{Checkpoint, DataStore, FromSnapshotWithParams, Snapshot}; +use data::{Checkpoint, FromSnapshotWithParams, Snapshot}; use enclave_core::{ Die, E3id, EnclaveEvent, EventBus, KeyshareCreated, OrderedSet, PublicKeyAggregated, Seed, }; @@ -10,6 +10,8 @@ use sortition::{GetHasNode, GetNodes, Sortition}; use std::sync::Arc; use tracing::error; +use crate::PublicKeyAggregatorRepository; + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum PublicKeyAggregatorState { Collecting { @@ -53,7 +55,7 @@ struct NotifyNetwork { pub struct PublicKeyAggregator { fhe: Arc, bus: Addr, - store: DataStore, + store: PublicKeyAggregatorRepository, sortition: Addr, e3_id: E3id, state: PublicKeyAggregatorState, @@ -63,7 +65,7 @@ pub struct PublicKeyAggregator { pub struct PublicKeyAggregatorParams { pub fhe: Arc, pub bus: Addr, - pub store: DataStore, + pub store: PublicKeyAggregatorRepository, pub sortition: Addr, pub e3_id: E3id, pub src_chain_id: u64, @@ -261,7 +263,8 @@ impl FromSnapshotWithParams for PublicKeyAggregator { } impl Checkpoint for PublicKeyAggregator { - fn get_store(&self) -> data::DataStore { + type Repository = PublicKeyAggregatorRepository; + fn get_store(&self) -> PublicKeyAggregatorRepository { self.store.clone() } } diff --git a/packages/ciphernode/aggregator/src/publickey_repository.rs b/packages/ciphernode/aggregator/src/publickey_repository.rs new file mode 100644 index 00000000..1b4613ba --- /dev/null +++ b/packages/ciphernode/aggregator/src/publickey_repository.rs @@ -0,0 +1,29 @@ +use crate::PublicKeyAggregatorState; +use async_trait::async_trait; +use data::{DataStore, Repository}; +use enclave_core::E3id; + +#[derive(Clone)] +pub struct PublicKeyAggregatorRepository { + store: DataStore, +} + +#[async_trait] +impl Repository for PublicKeyAggregatorRepository { + type State = PublicKeyAggregatorState; + fn store(&self) -> DataStore { + self.store.clone() + } +} + +pub trait PublicKeyAggregatorRepositoryFactory { + fn publickey(&self, e3_id: &E3id) -> PublicKeyAggregatorRepository; +} + +impl PublicKeyAggregatorRepositoryFactory for DataStore { + fn publickey(&self, e3_id: &E3id) -> PublicKeyAggregatorRepository { + PublicKeyAggregatorRepository { + store: self.scope(format!("//publickey/{e3_id}")), + } + } +} diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index fdf0a5cc..a8c8f1d6 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -33,6 +33,7 @@ impl Get { } } +/// Generate proxy for the DB #[derive(Clone)] pub struct DataStore { scope: Vec, diff --git a/packages/ciphernode/data/src/lib.rs b/packages/ciphernode/data/src/lib.rs index 285a095a..38e2c077 100644 --- a/packages/ciphernode/data/src/lib.rs +++ b/packages/ciphernode/data/src/lib.rs @@ -1,9 +1,11 @@ mod data_store; mod in_mem; mod into_key; +mod repository; mod snapshot; pub use data_store::*; pub use in_mem::*; pub use into_key::IntoKey; +pub use repository::*; pub use snapshot::*; diff --git a/packages/ciphernode/data/src/repository.rs b/packages/ciphernode/data/src/repository.rs new file mode 100644 index 00000000..a3cf77a2 --- /dev/null +++ b/packages/ciphernode/data/src/repository.rs @@ -0,0 +1,18 @@ +use anyhow::Result; +use async_trait::async_trait; + +use crate::DataStore; + +#[async_trait] +pub trait Repository { + type State: for<'de> serde::Deserialize<'de> + serde::Serialize; + fn store(&self) -> DataStore; + + async fn read(&self) -> Result> { + self.store().read().await + } + + fn write(&self, value: &Self::State) { + self.store().write(value) + } +} diff --git a/packages/ciphernode/data/src/snapshot.rs b/packages/ciphernode/data/src/snapshot.rs index 7204652a..3c7f5574 100644 --- a/packages/ciphernode/data/src/snapshot.rs +++ b/packages/ciphernode/data/src/snapshot.rs @@ -1,4 +1,4 @@ -use crate::DataStore; +use crate::{DataStore, Repository}; use anyhow::Result; use async_trait::async_trait; use serde::{de::DeserializeOwned, Serialize}; @@ -19,12 +19,13 @@ where /// This trait enables the self type to checkpoint its state pub trait Checkpoint: Snapshot { + type Repository: Repository; /// Declare the DataStore instance available on the object - fn get_store(&self) -> DataStore; + fn get_store(&self) -> Self::Repository; /// Write the current snapshot to the DataStore provided by `get_store()` at the object's id returned by `get_id()` fn checkpoint(&self) { - self.get_store().write(self.snapshot()); + self.get_store().write(&self.snapshot()); } } diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 63bd73e3..c9bbbd40 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -10,7 +10,7 @@ use p2p::P2p; use rand::SeedableRng; use rand_chacha::rand_core::OsRng; use router::{E3RequestRouter, FheFeature, PlaintextAggregatorFeature, PublicKeyAggregatorFeature}; -use sortition::Sortition; +use sortition::{Sortition, SortitionRepositoryFactory}; use std::sync::{Arc, Mutex}; use test_helpers::{PlaintextWriter, PublicKeyWriter}; use tokio::task::JoinHandle; @@ -53,7 +53,7 @@ impl MainAggregator { )); let store = DataStore::from_in_mem(InMemStore::new(true).start()); - let sortition = Sortition::attach(bus.clone(), store.clone()); + let sortition = Sortition::attach(bus.clone(), store.sortition()); let signer = pull_eth_signer_from_env("PRIVATE_KEY").await?; for chain in config .chains diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 98040d07..d0846cb9 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -9,7 +9,7 @@ use p2p::P2p; use rand::SeedableRng; use rand_chacha::rand_core::OsRng; use router::{CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature}; -use sortition::Sortition; +use sortition::{Sortition, SortitionRepositoryFactory}; use std::sync::{Arc, Mutex}; use tokio::task::JoinHandle; @@ -59,7 +59,7 @@ impl MainCiphernode { let bus = EventBus::new(true).start(); // TODO: switch to Sled actor let data = DataStore::from_in_mem(InMemStore::new(true).start()); - let sortition = Sortition::attach(bus.clone(), data.clone()); + let sortition = Sortition::attach(bus.clone(), data.sortition()); let selector = CiphernodeSelector::attach(bus.clone(), sortition.clone(), &address.to_string()); diff --git a/packages/ciphernode/fhe/src/lib.rs b/packages/ciphernode/fhe/src/lib.rs index 75b9bd75..00740ebc 100644 --- a/packages/ciphernode/fhe/src/lib.rs +++ b/packages/ciphernode/fhe/src/lib.rs @@ -1,5 +1,7 @@ mod fhe; +mod repository; mod utils; pub use fhe::*; +pub use repository::*; pub use utils::*; diff --git a/packages/ciphernode/fhe/src/repository.rs b/packages/ciphernode/fhe/src/repository.rs new file mode 100644 index 00000000..6c927ec2 --- /dev/null +++ b/packages/ciphernode/fhe/src/repository.rs @@ -0,0 +1,30 @@ +use async_trait::async_trait; +use data::{DataStore, Repository}; +use enclave_core::E3id; + +use crate::FheSnapshot; + +#[derive(Clone)] +pub struct FheRepository { + store: DataStore, +} + +#[async_trait] +impl Repository for FheRepository { + type State = FheSnapshot; + fn store(&self) -> DataStore { + self.store.clone() + } +} + +pub trait FheRepositoryFactory { + fn fhe(&self, e3_id: &E3id) -> FheRepository; +} + +impl FheRepositoryFactory for DataStore { + fn fhe(&self, e3_id: &E3id) -> FheRepository { + FheRepository { + store: self.scope(format!("//fhe/{e3_id}")), + } + } +} diff --git a/packages/ciphernode/keyshare/src/keyshare.rs b/packages/ciphernode/keyshare/src/keyshare.rs index ff680da0..7580b7c2 100644 --- a/packages/ciphernode/keyshare/src/keyshare.rs +++ b/packages/ciphernode/keyshare/src/keyshare.rs @@ -1,7 +1,7 @@ use actix::prelude::*; use anyhow::{anyhow, Result}; use async_trait::async_trait; -use data::{Checkpoint, DataStore, FromSnapshotWithParams, Snapshot}; +use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; use enclave_core::{ BusError, CiphernodeSelected, CiphertextOutputPublished, DecryptionshareCreated, Die, EnclaveErrorType, EnclaveEvent, EventBus, FromError, KeyshareCreated, @@ -10,9 +10,11 @@ use fhe::{DecryptCiphertext, Fhe}; use serde::{Deserialize, Serialize}; use std::sync::Arc; +use crate::KeyshareRepository; + pub struct Keyshare { fhe: Arc, - store: DataStore, + store: KeyshareRepository, bus: Addr, secret: Option>, address: String, @@ -24,7 +26,7 @@ impl Actor for Keyshare { pub struct KeyshareParams { pub bus: Addr, - pub store: DataStore, + pub store: KeyshareRepository, pub fhe: Arc, pub address: String, } @@ -57,7 +59,8 @@ impl Snapshot for Keyshare { } impl Checkpoint for Keyshare { - fn get_store(&self) -> DataStore { + type Repository = KeyshareRepository; + fn get_store(&self) -> KeyshareRepository { self.store.clone() } } diff --git a/packages/ciphernode/keyshare/src/lib.rs b/packages/ciphernode/keyshare/src/lib.rs index 46e4b5c9..7192e194 100644 --- a/packages/ciphernode/keyshare/src/lib.rs +++ b/packages/ciphernode/keyshare/src/lib.rs @@ -1,2 +1,4 @@ mod keyshare; +mod repository; pub use keyshare::*; +pub use repository::*; diff --git a/packages/ciphernode/keyshare/src/repository.rs b/packages/ciphernode/keyshare/src/repository.rs index ed8b85f1..b9cb2198 100644 --- a/packages/ciphernode/keyshare/src/repository.rs +++ b/packages/ciphernode/keyshare/src/repository.rs @@ -1,4 +1,29 @@ -use data::{Repository,DataStore}; use crate::keyshare::KeyshareState; +use async_trait::async_trait; +use data::{DataStore, Repository}; +use enclave_core::E3id; +#[derive(Clone)] +pub struct KeyshareRepository { + store: DataStore, +} +#[async_trait] +impl Repository for KeyshareRepository { + type State = KeyshareState; + fn store(&self) -> DataStore { + self.store.clone() + } +} + +pub trait KeyshareRepositoryFactory { + fn keyshare(&self, e3_id: &E3id) -> KeyshareRepository; +} + +impl KeyshareRepositoryFactory for DataStore { + fn keyshare(&self, e3_id: &E3id) -> KeyshareRepository { + KeyshareRepository { + store: self.scope(format!("//keyshare/{e3_id}")), + } + } +} diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index ab210143..833a54c5 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -1,6 +1,8 @@ +use crate::CommitteeMetaRepositoryFactory; use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot}; use anyhow::*; use async_trait::async_trait; +use data::{Checkpoint, Repository}; use enclave_core::{E3Requested, EnclaveEvent, Seed}; #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] @@ -33,13 +35,12 @@ impl E3Feature for CommitteeMetaFeature { } = data.clone(); // Meta doesn't implement Checkpoint so we are going to store it manually - let meta_id = format!("//meta/{e3_id}"); let meta = CommitteeMeta { threshold_m, seed, src_chain_id, }; - ctx.get_store().scope(&meta_id).write(meta.clone()); + ctx.get_store().store().meta(&e3_id).write(&meta); let _ = ctx.set_meta(meta); } @@ -49,12 +50,14 @@ impl E3Feature for CommitteeMetaFeature { snapshot: &E3RequestContextSnapshot, ) -> Result<()> { // No ID on the snapshot -> bail - let Some(id) = snapshot.meta.clone() else { + if !snapshot.meta { return Ok(()); }; + let repository = ctx.get_store().store().meta(&ctx.e3_id); + // No Snapshot returned from the store -> bail - let Some(value) = ctx.store.scope(&id).read().await? else { + let Some(value) = repository.read().await? else { return Ok(()); }; diff --git a/packages/ciphernode/router/src/committee_repository.rs b/packages/ciphernode/router/src/committee_repository.rs new file mode 100644 index 00000000..05498585 --- /dev/null +++ b/packages/ciphernode/router/src/committee_repository.rs @@ -0,0 +1,30 @@ +use async_trait::async_trait; +use data::{DataStore, Repository}; +use enclave_core::E3id; + +use crate::CommitteeMeta; + +#[derive(Clone)] +pub struct CommitteeMetaRepository { + store: DataStore, +} + +#[async_trait] +impl Repository for CommitteeMetaRepository { + type State = CommitteeMeta; + fn store(&self) -> DataStore { + self.store.clone() + } +} + +pub trait CommitteeMetaRepositoryFactory { + fn meta(&self, e3_id: &E3id) -> CommitteeMetaRepository; +} + +impl CommitteeMetaRepositoryFactory for DataStore { + fn meta(&self, e3_id: &E3id) -> CommitteeMetaRepository { + CommitteeMetaRepository { + store: self.scope(format!("//meta/{e3_id}")), + } + } +} diff --git a/packages/ciphernode/router/src/context.rs b/packages/ciphernode/router/src/context.rs new file mode 100644 index 00000000..26734727 --- /dev/null +++ b/packages/ciphernode/router/src/context.rs @@ -0,0 +1,179 @@ +use std::sync::Arc; + +use crate::{CommitteeMeta, E3ContextRepository, E3Feature, EventBuffer}; +use actix::{Addr, Recipient}; +use aggregator::{PlaintextAggregator, PublicKeyAggregator}; +use anyhow::Result; +use async_trait::async_trait; +use data::{Checkpoint, FromSnapshotWithParams, Snapshot}; +use enclave_core::{E3id, EnclaveEvent}; +use fhe::Fhe; +use keyshare::Keyshare; +use serde::{Deserialize, Serialize}; + +/// Context that is set to each event hook. Hooks can use this context to gather dependencies if +/// they need to instantiate struct instances or actors. +pub struct E3RequestContext { + pub e3_id: E3id, + pub keyshare: Option>, + pub fhe: Option>, + pub plaintext: Option>, + pub publickey: Option>, + pub meta: Option, + pub store: E3ContextRepository, +} + +#[derive(Serialize, Deserialize)] +pub struct E3RequestContextSnapshot { + pub keyshare: bool, + pub e3_id: E3id, + pub fhe: bool, + pub plaintext: bool, + pub publickey: bool, + pub meta: bool, +} + +pub struct E3RequestContextParams { + pub store: E3ContextRepository, + pub e3_id: E3id, + pub features: Arc>>, +} + +impl E3RequestContext { + pub fn from_params(params: E3RequestContextParams) -> Self { + Self { + e3_id: params.e3_id, + store: params.store, + fhe: None, + keyshare: None, + meta: None, + plaintext: None, + publickey: None, + } + } + + fn recipients(&self) -> Vec<(String, Option>)> { + vec![ + ( + "keyshare".to_owned(), + self.keyshare.clone().map(|addr| addr.into()), + ), + ( + "plaintext".to_owned(), + self.plaintext.clone().map(|addr| addr.into()), + ), + ( + "publickey".to_owned(), + self.publickey.clone().map(|addr| addr.into()), + ), + ] + } + + pub fn forward_message(&self, msg: &EnclaveEvent, buffer: &mut EventBuffer) { + self.recipients().into_iter().for_each(|(key, recipient)| { + if let Some(act) = recipient { + act.do_send(msg.clone()); + for m in buffer.take(&key) { + act.do_send(m); + } + } else { + buffer.add(&key, msg.clone()); + } + }); + } + + /// Accept a DataStore ID and a Keystore actor address + pub fn set_keyshare(&mut self, value: Addr) { + self.keyshare = Some(value); + self.checkpoint(); + } + + /// Accept a DataStore ID and a Keystore actor address + pub fn set_plaintext(&mut self, value: Addr) { + self.plaintext = Some(value); + self.checkpoint(); + } + + /// Accept a DataStore ID and a Keystore actor address + pub fn set_publickey(&mut self, value: Addr) { + self.publickey = Some(value); + self.checkpoint(); + } + + /// Accept a DataStore ID and an Arc instance of the Fhe wrapper + pub fn set_fhe(&mut self, value: Arc) { + self.fhe = Some(value.clone()); + self.checkpoint(); + } + + /// Accept a Datastore ID and a metadata object + pub fn set_meta(&mut self, value: CommitteeMeta) { + self.meta = Some(value.clone()); + self.checkpoint(); + } + + pub fn get_keyshare(&self) -> Option<&Addr> { + self.keyshare.as_ref() + } + + pub fn get_plaintext(&self) -> Option<&Addr> { + self.plaintext.as_ref() + } + + pub fn get_publickey(&self) -> Option<&Addr> { + self.publickey.as_ref() + } + + pub fn get_fhe(&self) -> Option<&Arc> { + self.fhe.as_ref() + } + + pub fn get_meta(&self) -> Option<&CommitteeMeta> { + self.meta.as_ref() + } +} + +#[async_trait] +impl Snapshot for E3RequestContext { + type Snapshot = E3RequestContextSnapshot; + + fn snapshot(&self) -> Self::Snapshot { + Self::Snapshot { + e3_id: self.e3_id.clone(), + meta: self.meta.is_some(), + fhe: self.fhe.is_some(), + publickey: self.publickey.is_some(), + plaintext: self.plaintext.is_some(), + keyshare: self.keyshare.is_some(), + } + } +} + +#[async_trait] +impl FromSnapshotWithParams for E3RequestContext { + type Params = E3RequestContextParams; + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + let mut ctx = Self { + e3_id: params.e3_id, + store: params.store, + fhe: None, + keyshare: None, + meta: None, + plaintext: None, + publickey: None, + }; + + for feature in params.features.iter() { + feature.hydrate(&mut ctx, &snapshot).await?; + } + + Ok(ctx) + } +} + +impl Checkpoint for E3RequestContext { + type Repository = E3ContextRepository; + fn get_store(&self) -> E3ContextRepository { + self.store.clone() + } +} diff --git a/packages/ciphernode/router/src/context_repository.rs b/packages/ciphernode/router/src/context_repository.rs new file mode 100644 index 00000000..33ab16b2 --- /dev/null +++ b/packages/ciphernode/router/src/context_repository.rs @@ -0,0 +1,30 @@ +use async_trait::async_trait; +use data::{DataStore, Repository}; +use enclave_core::E3id; + +use crate::E3RequestContextSnapshot; + +#[derive(Clone)] +pub struct E3ContextRepository { + store: DataStore, +} + +#[async_trait] +impl Repository for E3ContextRepository { + type State = E3RequestContextSnapshot; + fn store(&self) -> DataStore { + self.store.clone() + } +} + +pub trait E3ContextRepositoryFactory { + fn context(&self, e3_id: &E3id) -> E3ContextRepository; +} + +impl E3ContextRepositoryFactory for DataStore { + fn context(&self, e3_id: &E3id) -> E3ContextRepository { + E3ContextRepository { + store: self.scope(format!("//context/{e3_id}")), + } + } +} diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index df2996e8..ea1bdecc 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -1,19 +1,20 @@ use crate::CommitteeMetaFeature; - -use super::CommitteeMeta; -use actix::{Actor, Addr, Context, Handler, Recipient}; -use aggregator::PlaintextAggregator; -use aggregator::PublicKeyAggregator; +use crate::E3ContextRepositoryFactory; +use crate::E3RequestContext; +use crate::E3RequestContextParams; +use crate::E3RequestContextSnapshot; +use crate::E3RouterRepository; +use crate::E3RouterRepositoryFactory; +use actix::{Actor, Addr, Context, Handler}; use anyhow::*; use async_trait::async_trait; use data::Checkpoint; use data::DataStore; use data::FromSnapshotWithParams; +use data::Repository; use data::Snapshot; use enclave_core::E3RequestComplete; use enclave_core::{E3id, EnclaveEvent, EventBus, Subscribe}; -use fhe::Fhe; -use keyshare::Keyshare; use serde::Deserialize; use serde::Serialize; use std::collections::HashSet; @@ -21,7 +22,7 @@ use std::{collections::HashMap, sync::Arc}; /// Helper class to buffer events for downstream instances incase events arrive in the wrong order #[derive(Default)] -struct EventBuffer { +pub struct EventBuffer { buffer: HashMap>, } @@ -37,191 +38,6 @@ impl EventBuffer { .unwrap_or_default() } } - -/// Context that is set to each event hook. Hooks can use this context to gather dependencies if -/// they need to instantiate struct instances or actors. -pub struct E3RequestContext { - pub e3_id: E3id, - pub keyshare: Option>, - pub fhe: Option>, - pub plaintext: Option>, - pub publickey: Option>, - pub meta: Option, - pub store: DataStore, -} - -#[derive(Serialize, Deserialize)] -pub struct E3RequestContextSnapshot { - pub keyshare: Option, - pub fhe: Option, - pub plaintext: Option, - pub publickey: Option, - pub meta: Option, -} - -pub struct E3RequestContextParams { - pub store: DataStore, - pub e3_id: E3id, - pub features: Arc>>, -} - -impl E3RequestContext { - pub fn from_params(params: E3RequestContextParams) -> Self { - Self { - e3_id: params.e3_id, - store: params.store, - fhe: None, - keyshare: None, - meta: None, - plaintext: None, - publickey: None, - } - } - - fn recipients(&self) -> Vec<(String, Option>)> { - vec![ - ( - "keyshare".to_owned(), - self.keyshare.clone().map(|addr| addr.into()), - ), - ( - "plaintext".to_owned(), - self.plaintext.clone().map(|addr| addr.into()), - ), - ( - "publickey".to_owned(), - self.publickey.clone().map(|addr| addr.into()), - ), - ] - } - - fn forward_message(&self, msg: &EnclaveEvent, buffer: &mut EventBuffer) { - self.recipients().into_iter().for_each(|(key, recipient)| { - if let Some(act) = recipient { - act.do_send(msg.clone()); - for m in buffer.take(&key) { - act.do_send(m); - } - } else { - buffer.add(&key, msg.clone()); - } - }); - } - - /// Accept a DataStore ID and a Keystore actor address - pub fn set_keyshare(&mut self, value: Addr) { - self.keyshare = Some(value); - self.checkpoint(); - } - - /// Accept a DataStore ID and a Keystore actor address - pub fn set_plaintext(&mut self, value: Addr) { - self.plaintext = Some(value); - self.checkpoint(); - } - - /// Accept a DataStore ID and a Keystore actor address - pub fn set_publickey(&mut self, value: Addr) { - self.publickey = Some(value); - self.checkpoint(); - } - - /// Accept a DataStore ID and an Arc instance of the Fhe wrapper - pub fn set_fhe(&mut self, value: Arc) { - self.fhe = Some(value.clone()); - self.checkpoint(); - } - - /// Accept a Datastore ID and a metadata object - pub fn set_meta(&mut self, value: CommitteeMeta) { - self.meta = Some(value.clone()); - self.checkpoint(); - } - - pub fn get_keyshare(&self) -> Option<&Addr> { - self.keyshare.as_ref() - } - - pub fn get_plaintext(&self) -> Option<&Addr> { - self.plaintext.as_ref() - } - - pub fn get_publickey(&self) -> Option<&Addr> { - self.publickey.as_ref() - } - - pub fn get_fhe(&self) -> Option<&Arc> { - self.fhe.as_ref() - } - - pub fn get_meta(&self) -> Option<&CommitteeMeta> { - self.meta.as_ref() - } - - pub fn get_store(&self) -> DataStore { - self.store.clone() - } -} - -#[async_trait] -impl Snapshot for E3RequestContext { - type Snapshot = E3RequestContextSnapshot; - - fn snapshot(&self) -> Self::Snapshot { - let e3_id = self.e3_id.clone(); - let meta = self.meta.as_ref().map(|_| format!("//meta/{e3_id}")); - let fhe = self.fhe.as_ref().map(|_| format!("//fhe/{e3_id}")); - let publickey = self - .publickey - .as_ref() - .map(|_| format!("//publickey/{e3_id}")); - let plaintext = self - .plaintext - .as_ref() - .map(|_| format!("//plaintext/{e3_id}")); - let keyshare = self - .keyshare - .as_ref() - .map(|_| format!("//keyshare/{e3_id}")); - - Self::Snapshot { - meta, - fhe, - publickey, - plaintext, - keyshare, - } - } -} - -#[async_trait] -impl FromSnapshotWithParams for E3RequestContext { - type Params = E3RequestContextParams; - async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { - let mut ctx = Self { - e3_id: params.e3_id, - store: params.store, - fhe: None, - keyshare: None, - meta: None, - plaintext: None, - publickey: None, - }; - - for feature in params.features.iter() { - feature.hydrate(&mut ctx, &snapshot).await?; - } - - Ok(ctx) - } -} - -impl Checkpoint for E3RequestContext { - fn get_store(&self) -> DataStore { - self.store.clone() - } -} - /// Format of the hook that needs to be passed to E3RequestRouter // pub type EventHook = Box; // pub type Hydrator = Box; @@ -249,13 +65,13 @@ pub struct E3RequestRouter { features: Arc>>, buffer: EventBuffer, bus: Addr, - store: DataStore, + store: E3RouterRepository, } pub struct E3RequestRouterParams { features: Arc>>, bus: Addr, - store: DataStore, + store: E3RouterRepository, } impl E3RequestRouter { @@ -263,7 +79,7 @@ impl E3RequestRouter { let builder = E3RequestRouterBuilder { bus, features: vec![], - store, + store: store.router(), }; // Everything needs the committe meta factory so adding it here by default @@ -299,11 +115,11 @@ impl Handler for E3RequestRouter { // TODO: Log warning that e3 event was received for completed e3_id return; } - + let store = self.get_store().store(); let context = self.contexts.entry(e3_id.clone()).or_insert_with(|| { E3RequestContext::from_params(E3RequestContextParams { e3_id: e3_id.clone(), - store: self.store.scope(&format!("//context/{e3_id}")), + store: store.context(&e3_id), features: self.features.clone(), }) }); @@ -359,7 +175,8 @@ impl Snapshot for E3RequestRouter { } impl Checkpoint for E3RequestRouter { - fn get_store(&self) -> DataStore { + type Repository = E3RouterRepository; + fn get_store(&self) -> E3RouterRepository { self.store.clone() } } @@ -370,13 +187,9 @@ impl FromSnapshotWithParams for E3RequestRouter { async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { let mut contexts = HashMap::new(); + let store = params.store.store(); for e3_id in snapshot.contexts { - let Some(ctx_snapshot) = params - .store - .base(format!("//context/{e3_id}")) - .read() - .await? - else { + let Some(ctx_snapshot) = store.context(&e3_id).read().await? else { continue; }; @@ -384,7 +197,7 @@ impl FromSnapshotWithParams for E3RequestRouter { e3_id.clone(), E3RequestContext::from_snapshot( E3RequestContextParams { - store: params.store.clone(), + store: store.context(&e3_id), e3_id: e3_id.clone(), features: params.features.clone(), }, @@ -409,7 +222,7 @@ impl FromSnapshotWithParams for E3RequestRouter { pub struct E3RequestRouterBuilder { pub bus: Addr, pub features: Vec>, - pub store: DataStore, + pub store: E3RouterRepository, } impl E3RequestRouterBuilder { @@ -419,7 +232,7 @@ impl E3RequestRouterBuilder { } pub async fn build(self) -> Result> { - let snapshot: Option = self.store.base("//router").read().await?; + let snapshot: Option = self.store.store().router().read().await?; let params = E3RequestRouterParams { features: self.features.into(), bus: self.bus.clone(), diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index 92dde57f..5ae3fbc9 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -1,15 +1,16 @@ use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot}; use actix::{Actor, Addr}; use aggregator::{ - PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorState, PublicKeyAggregator, - PublicKeyAggregatorParams, PublicKeyAggregatorState, + PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorRepositoryFactory, + PlaintextAggregatorState, PublicKeyAggregator, PublicKeyAggregatorParams, + PublicKeyAggregatorRepositoryFactory, PublicKeyAggregatorState, }; use anyhow::{anyhow, Result}; use async_trait::async_trait; -use data::{FromSnapshotWithParams, Snapshot}; +use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; use enclave_core::{BusError, E3Requested, EnclaveErrorType, EnclaveEvent, EventBus}; -use fhe::{Fhe, SharedRng}; -use keyshare::{Keyshare, KeyshareParams}; +use fhe::{Fhe, FheRepositoryFactory, SharedRng}; +use keyshare::{Keyshare, KeyshareParams, KeyshareRepositoryFactory}; use sortition::Sortition; use std::sync::Arc; @@ -40,9 +41,8 @@ impl E3Feature for FheFeature { } = data.clone(); // FHE doesn't implement Checkpoint so we are going to store it manually - let fhe_id = format!("//fhe/{e3_id}"); let fhe = Arc::new(Fhe::from_encoded(¶ms, seed, self.rng.clone()).unwrap()); - ctx.get_store().scope(&fhe_id).write(fhe.snapshot()); + ctx.get_store().store().fhe(&e3_id).write(&fhe.snapshot()); let _ = ctx.set_fhe(fhe); } @@ -52,12 +52,12 @@ impl E3Feature for FheFeature { snapshot: &E3RequestContextSnapshot, ) -> Result<()> { // No ID on the snapshot -> bail - let Some(id) = snapshot.fhe.clone() else { + if !snapshot.fhe { return Ok(()); }; // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.scope(&id).read().await? else { + let Some(snap) = ctx.get_store().store().fhe(&ctx.e3_id).read().await? else { return Ok(()); }; @@ -97,12 +97,10 @@ impl E3Feature for KeyshareFeature { let e3_id = data.clone().e3_id; - let ks_id = format!("//keystore/{e3_id}"); - let _ = ctx.set_keyshare( Keyshare::new(KeyshareParams { bus: self.bus.clone(), - store: ctx.get_store().scope(&ks_id), + store: ctx.get_store().store().keyshare(&e3_id), fhe: fhe.clone(), address: self.address.clone(), }) @@ -116,12 +114,14 @@ impl E3Feature for KeyshareFeature { snapshot: &E3RequestContextSnapshot, ) -> Result<()> { // No ID on the snapshot -> bail - let Some(id) = snapshot.keyshare.clone() else { + if !snapshot.keyshare { return Ok(()); }; + let store = ctx.get_store().store().keyshare(&snapshot.e3_id); + // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.scope(&id).read().await? else { + let Some(snap) = store.read().await? else { return Ok(()); }; @@ -135,7 +135,7 @@ impl E3Feature for KeyshareFeature { KeyshareParams { fhe, bus: self.bus.clone(), - store: ctx.store.scope(&id), + store, address: self.address.clone(), }, snap, @@ -180,14 +180,12 @@ impl E3Feature for PlaintextAggregatorFeature { let e3_id = data.e3_id.clone(); - let id = &format!("//plaintext/{e3_id}"); - let _ = ctx.set_plaintext( PlaintextAggregator::new( PlaintextAggregatorParams { fhe: fhe.clone(), bus: self.bus.clone(), - store: ctx.get_store().scope(id), + store: ctx.get_store().store().plaintext(&e3_id), sortition: self.sortition.clone(), e3_id, src_chain_id: meta.src_chain_id, @@ -208,12 +206,14 @@ impl E3Feature for PlaintextAggregatorFeature { snapshot: &E3RequestContextSnapshot, ) -> Result<()> { // No ID on the snapshot -> bail - let Some(id) = snapshot.plaintext.clone() else { + if !snapshot.plaintext { return Ok(()); - }; + } + + let store = ctx.get_store().store().plaintext(&snapshot.e3_id); // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.scope(&id).read().await? else { + let Some(snap) = store.read().await? else { return Ok(()); }; @@ -230,7 +230,7 @@ impl E3Feature for PlaintextAggregatorFeature { PlaintextAggregatorParams { fhe: fhe.clone(), bus: self.bus.clone(), - store: ctx.get_store().scope(&id), + store, sortition: self.sortition.clone(), e3_id: ctx.e3_id.clone(), src_chain_id: meta.src_chain_id, @@ -276,14 +276,13 @@ impl E3Feature for PublicKeyAggregatorFeature { }; let e3_id = data.e3_id.clone(); - let id = &format!("//publickey/{e3_id}"); let _ = ctx.set_publickey( PublicKeyAggregator::new( PublicKeyAggregatorParams { fhe: fhe.clone(), bus: self.bus.clone(), - store: ctx.get_store().scope(id), + store: ctx.get_store().store().publickey(&e3_id), sortition: self.sortition.clone(), e3_id, src_chain_id: meta.src_chain_id, @@ -300,12 +299,14 @@ impl E3Feature for PublicKeyAggregatorFeature { snapshot: &E3RequestContextSnapshot, ) -> Result<()> { // No ID on the snapshot -> bail - let Some(id) = snapshot.publickey.clone() else { + if !snapshot.publickey { return Ok(()); }; + let repository = ctx.get_store().store().publickey(&ctx.e3_id); + // No Snapshot returned from the store -> bail - let Some(snap) = ctx.store.scope(&id).read().await? else { + let Some(snap) = repository.read().await? else { return Ok(()); }; @@ -322,7 +323,7 @@ impl E3Feature for PublicKeyAggregatorFeature { PublicKeyAggregatorParams { fhe: fhe.clone(), bus: self.bus.clone(), - store: ctx.get_store().scope(&id), + store: repository, sortition: self.sortition.clone(), e3_id: ctx.e3_id.clone(), src_chain_id: meta.src_chain_id, diff --git a/packages/ciphernode/router/src/lib.rs b/packages/ciphernode/router/src/lib.rs index 1f9fae2f..bb58b8aa 100644 --- a/packages/ciphernode/router/src/lib.rs +++ b/packages/ciphernode/router/src/lib.rs @@ -1,9 +1,17 @@ mod ciphernode_selector; mod committee_meta; +mod committee_repository; +mod context; +mod context_repository; mod e3_request_router; mod hooks; +mod router_repository; pub use ciphernode_selector::*; pub use committee_meta::*; +pub use committee_repository::*; +pub use context::*; +pub use context_repository::*; pub use e3_request_router::*; pub use hooks::*; +pub use router_repository::*; diff --git a/packages/ciphernode/router/src/router_repository.rs b/packages/ciphernode/router/src/router_repository.rs new file mode 100644 index 00000000..69fcfe67 --- /dev/null +++ b/packages/ciphernode/router/src/router_repository.rs @@ -0,0 +1,29 @@ +use async_trait::async_trait; +use data::{DataStore, Repository}; + +use crate::E3RequestRouterSnapshot; + +#[derive(Clone)] +pub struct E3RouterRepository { + store: DataStore, +} + +#[async_trait] +impl Repository for E3RouterRepository { + type State = E3RequestRouterSnapshot; + fn store(&self) -> DataStore { + self.store.clone() + } +} + +pub trait E3RouterRepositoryFactory { + fn router(&self) -> E3RouterRepository; +} + +impl E3RouterRepositoryFactory for DataStore { + fn router(&self) -> E3RouterRepository { + E3RouterRepository { + store: self.scope(format!("//router")), + } + } +} diff --git a/packages/ciphernode/sortition/src/lib.rs b/packages/ciphernode/sortition/src/lib.rs index 792f64fc..7113c06d 100644 --- a/packages/ciphernode/sortition/src/lib.rs +++ b/packages/ciphernode/sortition/src/lib.rs @@ -4,8 +4,10 @@ mod distance; mod index; +mod repository; mod sortition; pub use distance::*; pub use index::*; +pub use repository::*; pub use sortition::*; diff --git a/packages/ciphernode/sortition/src/repository.rs b/packages/ciphernode/sortition/src/repository.rs new file mode 100644 index 00000000..e10c4149 --- /dev/null +++ b/packages/ciphernode/sortition/src/repository.rs @@ -0,0 +1,37 @@ +use crate::SortitionModule; +use anyhow::*; +use async_trait::async_trait; +use data::{DataStore, Repository}; + +#[derive(Clone)] +pub struct SortitionRepository { + store: DataStore, +} + +#[async_trait] +impl Repository for SortitionRepository { + type State = SortitionModule; + async fn read(&self) -> Result> { + self.store.read().await + } + + fn write(&self, value: &SortitionModule) { + self.store.write(value) + } + + fn store(&self) -> DataStore { + self.store.clone() + } +} + +pub trait SortitionRepositoryFactory { + fn sortition(&self) -> SortitionRepository; +} + +impl SortitionRepositoryFactory for DataStore { + fn sortition(&self) -> SortitionRepository { + SortitionRepository { + store: self.scope(format!("//sortition")), + } + } +} diff --git a/packages/ciphernode/sortition/src/sortition.rs b/packages/ciphernode/sortition/src/sortition.rs index bd582537..97357c61 100644 --- a/packages/ciphernode/sortition/src/sortition.rs +++ b/packages/ciphernode/sortition/src/sortition.rs @@ -1,9 +1,9 @@ -use crate::DistanceSortition; +use crate::{DistanceSortition, SortitionRepository}; use actix::prelude::*; use alloy::primitives::Address; use anyhow::{anyhow, Result}; use async_trait::async_trait; -use data::{Checkpoint, DataStore, FromSnapshotWithParams, Snapshot}; +use data::{Checkpoint, FromSnapshotWithParams, Snapshot}; use enclave_core::{ BusError, CiphernodeAdded, CiphernodeRemoved, EnclaveErrorType, EnclaveEvent, EventBus, Seed, Subscribe, @@ -84,12 +84,12 @@ pub struct GetNodes; pub struct Sortition { list: SortitionModule, bus: Addr, - store: DataStore, + store: SortitionRepository, } pub struct SortitionParams { pub bus: Addr, - pub store: DataStore, + pub store: SortitionRepository, } impl Sortition { @@ -101,7 +101,7 @@ impl Sortition { } } - pub fn attach(bus: Addr, store: DataStore) -> Addr { + pub fn attach(bus: Addr, store: SortitionRepository) -> Addr { let addr = Sortition::new(SortitionParams { bus: bus.clone(), store, @@ -140,7 +140,8 @@ impl FromSnapshotWithParams for Sortition { } impl Checkpoint for Sortition { - fn get_store(&self) -> DataStore { + type Repository = SortitionRepository; + fn get_store(&self) -> SortitionRepository { self.store.clone() } } From 05e6d145cb14a8c87a64f1143dfcdbdbb3799235 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Fri, 18 Oct 2024 19:46:49 +1100 Subject: [PATCH 16/49] Setup repositries struct for finding repositories --- packages/ciphernode/aggregator/src/lib.rs | 4 -- .../aggregator/src/plaintext_aggregator.rs | 11 ++-- .../aggregator/src/plaintext_repository.rs | 29 --------- .../aggregator/src/publickey_aggregator.rs | 10 ++- .../aggregator/src/publickey_repository.rs | 29 --------- packages/ciphernode/data/src/repository.rs | 55 +++++++++++++--- packages/ciphernode/data/src/snapshot.rs | 5 +- .../ciphernode/enclave_node/src/aggregator.rs | 10 ++- .../ciphernode/enclave_node/src/ciphernode.rs | 14 ++-- packages/ciphernode/fhe/src/lib.rs | 2 - packages/ciphernode/fhe/src/repository.rs | 30 --------- packages/ciphernode/keyshare/src/keyshare.rs | 9 +-- packages/ciphernode/keyshare/src/lib.rs | 2 - .../ciphernode/keyshare/src/repository.rs | 29 --------- .../ciphernode/router/src/committee_meta.rs | 6 +- .../router/src/committee_repository.rs | 30 --------- packages/ciphernode/router/src/context.rs | 15 +++-- .../router/src/context_repository.rs | 30 --------- .../router/src/e3_request_router.rs | 36 ++++++----- packages/ciphernode/router/src/hooks.rs | 26 ++++---- packages/ciphernode/router/src/lib.rs | 8 +-- .../ciphernode/router/src/repositories.rs | 64 +++++++++++++++++++ .../router/src/router_repository.rs | 29 --------- packages/ciphernode/sortition/src/lib.rs | 2 - .../ciphernode/sortition/src/repository.rs | 37 ----------- .../ciphernode/sortition/src/sortition.rs | 13 ++-- .../tests/test_aggregation_and_decryption.rs | 5 +- 27 files changed, 192 insertions(+), 348 deletions(-) delete mode 100644 packages/ciphernode/aggregator/src/plaintext_repository.rs delete mode 100644 packages/ciphernode/aggregator/src/publickey_repository.rs delete mode 100644 packages/ciphernode/fhe/src/repository.rs delete mode 100644 packages/ciphernode/keyshare/src/repository.rs delete mode 100644 packages/ciphernode/router/src/committee_repository.rs delete mode 100644 packages/ciphernode/router/src/context_repository.rs create mode 100644 packages/ciphernode/router/src/repositories.rs delete mode 100644 packages/ciphernode/router/src/router_repository.rs delete mode 100644 packages/ciphernode/sortition/src/repository.rs diff --git a/packages/ciphernode/aggregator/src/lib.rs b/packages/ciphernode/aggregator/src/lib.rs index 59e3bbf6..fef3eb91 100644 --- a/packages/ciphernode/aggregator/src/lib.rs +++ b/packages/ciphernode/aggregator/src/lib.rs @@ -1,12 +1,8 @@ mod plaintext_aggregator; -mod plaintext_repository; mod publickey_aggregator; -mod publickey_repository; pub use plaintext_aggregator::{ PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorState, }; -pub use plaintext_repository::*; pub use publickey_aggregator::{ PublicKeyAggregator, PublicKeyAggregatorParams, PublicKeyAggregatorState, }; -pub use publickey_repository::*; diff --git a/packages/ciphernode/aggregator/src/plaintext_aggregator.rs b/packages/ciphernode/aggregator/src/plaintext_aggregator.rs index fd60bead..c65e8e55 100644 --- a/packages/ciphernode/aggregator/src/plaintext_aggregator.rs +++ b/packages/ciphernode/aggregator/src/plaintext_aggregator.rs @@ -1,7 +1,7 @@ use actix::prelude::*; use anyhow::Result; use async_trait::async_trait; -use data::{Checkpoint, FromSnapshotWithParams, Snapshot}; +use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; use enclave_core::{ DecryptionshareCreated, Die, E3id, EnclaveEvent, EventBus, OrderedSet, PlaintextAggregated, Seed, @@ -11,8 +11,6 @@ use sortition::{GetHasNode, Sortition}; use std::sync::Arc; use tracing::error; -use crate::plaintext_repository::PlaintextAggregatorRepository; - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum PlaintextAggregatorState { Collecting { @@ -52,7 +50,7 @@ struct ComputeAggregate { pub struct PlaintextAggregator { fhe: Arc, bus: Addr, - store: PlaintextAggregatorRepository, + store: Repository, sortition: Addr, e3_id: E3id, state: PlaintextAggregatorState, @@ -62,7 +60,7 @@ pub struct PlaintextAggregator { pub struct PlaintextAggregatorParams { pub fhe: Arc, pub bus: Addr, - pub store: PlaintextAggregatorRepository, + pub store: Repository, pub sortition: Addr, pub e3_id: E3id, pub src_chain_id: u64, @@ -238,8 +236,7 @@ impl FromSnapshotWithParams for PlaintextAggregator { } impl Checkpoint for PlaintextAggregator { - type Repository = PlaintextAggregatorRepository; - fn get_store(&self) -> PlaintextAggregatorRepository { + fn repository(&self) -> Repository { self.store.clone() } } diff --git a/packages/ciphernode/aggregator/src/plaintext_repository.rs b/packages/ciphernode/aggregator/src/plaintext_repository.rs deleted file mode 100644 index 4c19aeb1..00000000 --- a/packages/ciphernode/aggregator/src/plaintext_repository.rs +++ /dev/null @@ -1,29 +0,0 @@ -use crate::PlaintextAggregatorState; -use async_trait::async_trait; -use data::{DataStore, Repository}; -use enclave_core::E3id; - -#[derive(Clone)] -pub struct PlaintextAggregatorRepository { - store: DataStore, -} - -#[async_trait] -impl Repository for PlaintextAggregatorRepository { - type State = PlaintextAggregatorState; - fn store(&self) -> DataStore { - self.store.clone() - } -} - -pub trait PlaintextAggregatorRepositoryFactory { - fn plaintext(&self, e3_id: &E3id) -> PlaintextAggregatorRepository; -} - -impl PlaintextAggregatorRepositoryFactory for DataStore { - fn plaintext(&self, e3_id: &E3id) -> PlaintextAggregatorRepository { - PlaintextAggregatorRepository { - store: self.scope(format!("//plaintext/{e3_id}")), - } - } -} diff --git a/packages/ciphernode/aggregator/src/publickey_aggregator.rs b/packages/ciphernode/aggregator/src/publickey_aggregator.rs index 85a0dea4..81d98910 100644 --- a/packages/ciphernode/aggregator/src/publickey_aggregator.rs +++ b/packages/ciphernode/aggregator/src/publickey_aggregator.rs @@ -1,7 +1,7 @@ use actix::prelude::*; use anyhow::Result; use async_trait::async_trait; -use data::{Checkpoint, FromSnapshotWithParams, Snapshot}; +use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; use enclave_core::{ Die, E3id, EnclaveEvent, EventBus, KeyshareCreated, OrderedSet, PublicKeyAggregated, Seed, }; @@ -10,7 +10,6 @@ use sortition::{GetHasNode, GetNodes, Sortition}; use std::sync::Arc; use tracing::error; -use crate::PublicKeyAggregatorRepository; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum PublicKeyAggregatorState { @@ -55,7 +54,7 @@ struct NotifyNetwork { pub struct PublicKeyAggregator { fhe: Arc, bus: Addr, - store: PublicKeyAggregatorRepository, + store: Repository, sortition: Addr, e3_id: E3id, state: PublicKeyAggregatorState, @@ -65,7 +64,7 @@ pub struct PublicKeyAggregator { pub struct PublicKeyAggregatorParams { pub fhe: Arc, pub bus: Addr, - pub store: PublicKeyAggregatorRepository, + pub store: Repository, pub sortition: Addr, pub e3_id: E3id, pub src_chain_id: u64, @@ -263,8 +262,7 @@ impl FromSnapshotWithParams for PublicKeyAggregator { } impl Checkpoint for PublicKeyAggregator { - type Repository = PublicKeyAggregatorRepository; - fn get_store(&self) -> PublicKeyAggregatorRepository { + fn repository(&self) -> Repository { self.store.clone() } } diff --git a/packages/ciphernode/aggregator/src/publickey_repository.rs b/packages/ciphernode/aggregator/src/publickey_repository.rs deleted file mode 100644 index 1b4613ba..00000000 --- a/packages/ciphernode/aggregator/src/publickey_repository.rs +++ /dev/null @@ -1,29 +0,0 @@ -use crate::PublicKeyAggregatorState; -use async_trait::async_trait; -use data::{DataStore, Repository}; -use enclave_core::E3id; - -#[derive(Clone)] -pub struct PublicKeyAggregatorRepository { - store: DataStore, -} - -#[async_trait] -impl Repository for PublicKeyAggregatorRepository { - type State = PublicKeyAggregatorState; - fn store(&self) -> DataStore { - self.store.clone() - } -} - -pub trait PublicKeyAggregatorRepositoryFactory { - fn publickey(&self, e3_id: &E3id) -> PublicKeyAggregatorRepository; -} - -impl PublicKeyAggregatorRepositoryFactory for DataStore { - fn publickey(&self, e3_id: &E3id) -> PublicKeyAggregatorRepository { - PublicKeyAggregatorRepository { - store: self.scope(format!("//publickey/{e3_id}")), - } - } -} diff --git a/packages/ciphernode/data/src/repository.rs b/packages/ciphernode/data/src/repository.rs index a3cf77a2..c54de5bd 100644 --- a/packages/ciphernode/data/src/repository.rs +++ b/packages/ciphernode/data/src/repository.rs @@ -1,18 +1,55 @@ +use std::{marker::PhantomData, ops::Deref}; + use anyhow::Result; -use async_trait::async_trait; use crate::DataStore; -#[async_trait] -pub trait Repository { - type State: for<'de> serde::Deserialize<'de> + serde::Serialize; - fn store(&self) -> DataStore; +pub struct Repository { + store: DataStore, + _p: PhantomData, +} + +impl Repository { + pub fn new(store: DataStore) -> Self { + Self { + store, + _p: PhantomData, + } + } +} + +impl Deref for Repository{ + type Target = DataStore; + fn deref(&self) -> &Self::Target { + &self.store + } +} + +impl From> for DataStore { + fn from(value: Repository) -> Self { + value.store + } +} + +/// Clone without phantom data +impl Clone for Repository { + fn clone(&self) -> Self { + Self { + store: self.store.clone(), + _p: PhantomData, + } + } +} - async fn read(&self) -> Result> { - self.store().read().await +impl Repository +where + T: for<'de> serde::Deserialize<'de> + serde::Serialize, +{ + pub async fn read(&self) -> Result> { + self.store.read().await } - fn write(&self, value: &Self::State) { - self.store().write(value) + pub fn write(&self, value: &T) { + self.store.write(value) } } diff --git a/packages/ciphernode/data/src/snapshot.rs b/packages/ciphernode/data/src/snapshot.rs index 3c7f5574..f747a85a 100644 --- a/packages/ciphernode/data/src/snapshot.rs +++ b/packages/ciphernode/data/src/snapshot.rs @@ -19,13 +19,12 @@ where /// This trait enables the self type to checkpoint its state pub trait Checkpoint: Snapshot { - type Repository: Repository; /// Declare the DataStore instance available on the object - fn get_store(&self) -> Self::Repository; + fn repository(&self) -> Repository; /// Write the current snapshot to the DataStore provided by `get_store()` at the object's id returned by `get_id()` fn checkpoint(&self) { - self.get_store().write(&self.snapshot()); + self.repository().write(&self.snapshot()); } } diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index c9bbbd40..c0270ab4 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -9,8 +9,11 @@ use logger::SimpleLogger; use p2p::P2p; use rand::SeedableRng; use rand_chacha::rand_core::OsRng; -use router::{E3RequestRouter, FheFeature, PlaintextAggregatorFeature, PublicKeyAggregatorFeature}; -use sortition::{Sortition, SortitionRepositoryFactory}; +use router::{ + E3RequestRouter, FheFeature, PlaintextAggregatorFeature, PublicKeyAggregatorFeature, + Repositories, +}; +use sortition::Sortition; use std::sync::{Arc, Mutex}; use test_helpers::{PlaintextWriter, PublicKeyWriter}; use tokio::task::JoinHandle; @@ -53,7 +56,8 @@ impl MainAggregator { )); let store = DataStore::from_in_mem(InMemStore::new(true).start()); - let sortition = Sortition::attach(bus.clone(), store.sortition()); + let repositories: Repositories = store.clone().into(); + let sortition = Sortition::attach(bus.clone(), repositories.sortition()); let signer = pull_eth_signer_from_env("PRIVATE_KEY").await?; for chain in config .chains diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index d0846cb9..0671f855 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -8,8 +8,8 @@ use logger::SimpleLogger; use p2p::P2p; use rand::SeedableRng; use rand_chacha::rand_core::OsRng; -use router::{CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature}; -use sortition::{Sortition, SortitionRepositoryFactory}; +use router::{CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, Repositories}; +use sortition::Sortition; use std::sync::{Arc, Mutex}; use tokio::task::JoinHandle; @@ -58,8 +58,10 @@ impl MainCiphernode { )); let bus = EventBus::new(true).start(); // TODO: switch to Sled actor - let data = DataStore::from_in_mem(InMemStore::new(true).start()); - let sortition = Sortition::attach(bus.clone(), data.sortition()); + let store = DataStore::from_in_mem(InMemStore::new(true).start()); + let repositories: Repositories = store.clone().into(); + + let sortition = Sortition::attach(bus.clone(), repositories.sortition()); let selector = CiphernodeSelector::attach(bus.clone(), sortition.clone(), &address.to_string()); @@ -79,7 +81,7 @@ impl MainCiphernode { .await?; } - let e3_manager = E3RequestRouter::builder(bus.clone(), data.clone()) + let e3_manager = E3RequestRouter::builder(bus.clone(), store.clone()) .add_feature(FheFeature::create(rng)) .add_feature(KeyshareFeature::create(bus.clone(), &address.to_string())) .build() @@ -91,7 +93,7 @@ impl MainCiphernode { let nm = format!("CIPHER({})", &address.to_string()[0..5]); SimpleLogger::attach(&nm, bus.clone()); let main_addr = MainCiphernode::new( - address, bus, data, sortition, selector, p2p_addr, e3_manager, + address, bus, store, sortition, selector, p2p_addr, e3_manager, ) .start(); Ok((main_addr, join_handle)) diff --git a/packages/ciphernode/fhe/src/lib.rs b/packages/ciphernode/fhe/src/lib.rs index 00740ebc..75b9bd75 100644 --- a/packages/ciphernode/fhe/src/lib.rs +++ b/packages/ciphernode/fhe/src/lib.rs @@ -1,7 +1,5 @@ mod fhe; -mod repository; mod utils; pub use fhe::*; -pub use repository::*; pub use utils::*; diff --git a/packages/ciphernode/fhe/src/repository.rs b/packages/ciphernode/fhe/src/repository.rs deleted file mode 100644 index 6c927ec2..00000000 --- a/packages/ciphernode/fhe/src/repository.rs +++ /dev/null @@ -1,30 +0,0 @@ -use async_trait::async_trait; -use data::{DataStore, Repository}; -use enclave_core::E3id; - -use crate::FheSnapshot; - -#[derive(Clone)] -pub struct FheRepository { - store: DataStore, -} - -#[async_trait] -impl Repository for FheRepository { - type State = FheSnapshot; - fn store(&self) -> DataStore { - self.store.clone() - } -} - -pub trait FheRepositoryFactory { - fn fhe(&self, e3_id: &E3id) -> FheRepository; -} - -impl FheRepositoryFactory for DataStore { - fn fhe(&self, e3_id: &E3id) -> FheRepository { - FheRepository { - store: self.scope(format!("//fhe/{e3_id}")), - } - } -} diff --git a/packages/ciphernode/keyshare/src/keyshare.rs b/packages/ciphernode/keyshare/src/keyshare.rs index 7580b7c2..5f29a3b5 100644 --- a/packages/ciphernode/keyshare/src/keyshare.rs +++ b/packages/ciphernode/keyshare/src/keyshare.rs @@ -10,11 +10,9 @@ use fhe::{DecryptCiphertext, Fhe}; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use crate::KeyshareRepository; - pub struct Keyshare { fhe: Arc, - store: KeyshareRepository, + store: Repository, bus: Addr, secret: Option>, address: String, @@ -26,7 +24,7 @@ impl Actor for Keyshare { pub struct KeyshareParams { pub bus: Addr, - pub store: KeyshareRepository, + pub store: Repository, pub fhe: Arc, pub address: String, } @@ -59,8 +57,7 @@ impl Snapshot for Keyshare { } impl Checkpoint for Keyshare { - type Repository = KeyshareRepository; - fn get_store(&self) -> KeyshareRepository { + fn repository(&self) -> Repository{ self.store.clone() } } diff --git a/packages/ciphernode/keyshare/src/lib.rs b/packages/ciphernode/keyshare/src/lib.rs index 7192e194..46e4b5c9 100644 --- a/packages/ciphernode/keyshare/src/lib.rs +++ b/packages/ciphernode/keyshare/src/lib.rs @@ -1,4 +1,2 @@ mod keyshare; -mod repository; pub use keyshare::*; -pub use repository::*; diff --git a/packages/ciphernode/keyshare/src/repository.rs b/packages/ciphernode/keyshare/src/repository.rs deleted file mode 100644 index b9cb2198..00000000 --- a/packages/ciphernode/keyshare/src/repository.rs +++ /dev/null @@ -1,29 +0,0 @@ -use crate::keyshare::KeyshareState; -use async_trait::async_trait; -use data::{DataStore, Repository}; -use enclave_core::E3id; - -#[derive(Clone)] -pub struct KeyshareRepository { - store: DataStore, -} - -#[async_trait] -impl Repository for KeyshareRepository { - type State = KeyshareState; - fn store(&self) -> DataStore { - self.store.clone() - } -} - -pub trait KeyshareRepositoryFactory { - fn keyshare(&self, e3_id: &E3id) -> KeyshareRepository; -} - -impl KeyshareRepositoryFactory for DataStore { - fn keyshare(&self, e3_id: &E3id) -> KeyshareRepository { - KeyshareRepository { - store: self.scope(format!("//keyshare/{e3_id}")), - } - } -} diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index 833a54c5..0749f040 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -1,8 +1,6 @@ -use crate::CommitteeMetaRepositoryFactory; use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot}; use anyhow::*; use async_trait::async_trait; -use data::{Checkpoint, Repository}; use enclave_core::{E3Requested, EnclaveEvent, Seed}; #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] @@ -40,7 +38,7 @@ impl E3Feature for CommitteeMetaFeature { seed, src_chain_id, }; - ctx.get_store().store().meta(&e3_id).write(&meta); + ctx.repositories().meta(&e3_id).write(&meta); let _ = ctx.set_meta(meta); } @@ -54,7 +52,7 @@ impl E3Feature for CommitteeMetaFeature { return Ok(()); }; - let repository = ctx.get_store().store().meta(&ctx.e3_id); + let repository = ctx.repositories().meta(&ctx.e3_id); // No Snapshot returned from the store -> bail let Some(value) = repository.read().await? else { diff --git a/packages/ciphernode/router/src/committee_repository.rs b/packages/ciphernode/router/src/committee_repository.rs deleted file mode 100644 index 05498585..00000000 --- a/packages/ciphernode/router/src/committee_repository.rs +++ /dev/null @@ -1,30 +0,0 @@ -use async_trait::async_trait; -use data::{DataStore, Repository}; -use enclave_core::E3id; - -use crate::CommitteeMeta; - -#[derive(Clone)] -pub struct CommitteeMetaRepository { - store: DataStore, -} - -#[async_trait] -impl Repository for CommitteeMetaRepository { - type State = CommitteeMeta; - fn store(&self) -> DataStore { - self.store.clone() - } -} - -pub trait CommitteeMetaRepositoryFactory { - fn meta(&self, e3_id: &E3id) -> CommitteeMetaRepository; -} - -impl CommitteeMetaRepositoryFactory for DataStore { - fn meta(&self, e3_id: &E3id) -> CommitteeMetaRepository { - CommitteeMetaRepository { - store: self.scope(format!("//meta/{e3_id}")), - } - } -} diff --git a/packages/ciphernode/router/src/context.rs b/packages/ciphernode/router/src/context.rs index 26734727..81bcf4ca 100644 --- a/packages/ciphernode/router/src/context.rs +++ b/packages/ciphernode/router/src/context.rs @@ -1,11 +1,11 @@ use std::sync::Arc; -use crate::{CommitteeMeta, E3ContextRepository, E3Feature, EventBuffer}; +use crate::{CommitteeMeta, E3Feature, EventBuffer, Repositories}; use actix::{Addr, Recipient}; use aggregator::{PlaintextAggregator, PublicKeyAggregator}; use anyhow::Result; use async_trait::async_trait; -use data::{Checkpoint, FromSnapshotWithParams, Snapshot}; +use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; use enclave_core::{E3id, EnclaveEvent}; use fhe::Fhe; use keyshare::Keyshare; @@ -20,7 +20,7 @@ pub struct E3RequestContext { pub plaintext: Option>, pub publickey: Option>, pub meta: Option, - pub store: E3ContextRepository, + pub store: Repository, } #[derive(Serialize, Deserialize)] @@ -34,7 +34,7 @@ pub struct E3RequestContextSnapshot { } pub struct E3RequestContextParams { - pub store: E3ContextRepository, + pub store: Repository, pub e3_id: E3id, pub features: Arc>>, } @@ -82,6 +82,10 @@ impl E3RequestContext { }); } + pub fn repositories(&self) -> Repositories { + self.repository().into() + } + /// Accept a DataStore ID and a Keystore actor address pub fn set_keyshare(&mut self, value: Addr) { self.keyshare = Some(value); @@ -172,8 +176,7 @@ impl FromSnapshotWithParams for E3RequestContext { } impl Checkpoint for E3RequestContext { - type Repository = E3ContextRepository; - fn get_store(&self) -> E3ContextRepository { + fn repository(&self) -> Repository { self.store.clone() } } diff --git a/packages/ciphernode/router/src/context_repository.rs b/packages/ciphernode/router/src/context_repository.rs deleted file mode 100644 index 33ab16b2..00000000 --- a/packages/ciphernode/router/src/context_repository.rs +++ /dev/null @@ -1,30 +0,0 @@ -use async_trait::async_trait; -use data::{DataStore, Repository}; -use enclave_core::E3id; - -use crate::E3RequestContextSnapshot; - -#[derive(Clone)] -pub struct E3ContextRepository { - store: DataStore, -} - -#[async_trait] -impl Repository for E3ContextRepository { - type State = E3RequestContextSnapshot; - fn store(&self) -> DataStore { - self.store.clone() - } -} - -pub trait E3ContextRepositoryFactory { - fn context(&self, e3_id: &E3id) -> E3ContextRepository; -} - -impl E3ContextRepositoryFactory for DataStore { - fn context(&self, e3_id: &E3id) -> E3ContextRepository { - E3ContextRepository { - store: self.scope(format!("//context/{e3_id}")), - } - } -} diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index ea1bdecc..c4554a51 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -1,10 +1,8 @@ use crate::CommitteeMetaFeature; -use crate::E3ContextRepositoryFactory; use crate::E3RequestContext; use crate::E3RequestContextParams; use crate::E3RequestContextSnapshot; -use crate::E3RouterRepository; -use crate::E3RouterRepositoryFactory; +use crate::Repositories; use actix::{Actor, Addr, Context, Handler}; use anyhow::*; use async_trait::async_trait; @@ -65,21 +63,22 @@ pub struct E3RequestRouter { features: Arc>>, buffer: EventBuffer, bus: Addr, - store: E3RouterRepository, + store: Repository, } pub struct E3RequestRouterParams { features: Arc>>, bus: Addr, - store: E3RouterRepository, + store: Repository, } impl E3RequestRouter { pub fn builder(bus: Addr, store: DataStore) -> E3RequestRouterBuilder { + let repositories: Repositories = store.into(); let builder = E3RequestRouterBuilder { bus, features: vec![], - store: store.router(), + store: repositories.router(), }; // Everything needs the committe meta factory so adding it here by default @@ -115,11 +114,12 @@ impl Handler for E3RequestRouter { // TODO: Log warning that e3 event was received for completed e3_id return; } - let store = self.get_store().store(); + + let repositories: Repositories = self.repository().into(); let context = self.contexts.entry(e3_id.clone()).or_insert_with(|| { E3RequestContext::from_params(E3RequestContextParams { e3_id: e3_id.clone(), - store: store.context(&e3_id), + store: repositories.context(&e3_id), features: self.features.clone(), }) }); @@ -175,8 +175,7 @@ impl Snapshot for E3RequestRouter { } impl Checkpoint for E3RequestRouter { - type Repository = E3RouterRepository; - fn get_store(&self) -> E3RouterRepository { + fn repository(&self) -> Repository { self.store.clone() } } @@ -187,9 +186,10 @@ impl FromSnapshotWithParams for E3RequestRouter { async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { let mut contexts = HashMap::new(); - let store = params.store.store(); + + let repositories: Repositories = params.store.into(); for e3_id in snapshot.contexts { - let Some(ctx_snapshot) = store.context(&e3_id).read().await? else { + let Some(ctx_snapshot) = repositories.context(&e3_id).read().await? else { continue; }; @@ -197,7 +197,7 @@ impl FromSnapshotWithParams for E3RequestRouter { e3_id.clone(), E3RequestContext::from_snapshot( E3RequestContextParams { - store: store.context(&e3_id), + store: repositories.context(&e3_id), e3_id: e3_id.clone(), features: params.features.clone(), }, @@ -213,7 +213,7 @@ impl FromSnapshotWithParams for E3RequestRouter { features: params.features.into(), buffer: EventBuffer::default(), bus: params.bus, - store: params.store, + store: repositories.router(), }) } } @@ -222,7 +222,7 @@ impl FromSnapshotWithParams for E3RequestRouter { pub struct E3RequestRouterBuilder { pub bus: Addr, pub features: Vec>, - pub store: E3RouterRepository, + pub store: Repository, } impl E3RequestRouterBuilder { @@ -232,12 +232,14 @@ impl E3RequestRouterBuilder { } pub async fn build(self) -> Result> { - let snapshot: Option = self.store.store().router().read().await?; + let repositories: Repositories = self.store.into(); + let router_repo = repositories.router(); + let snapshot: Option = router_repo.read().await?; let params = E3RequestRouterParams { features: self.features.into(), bus: self.bus.clone(), - store: self.store, + store: router_repo, }; let e3r = match snapshot { diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index 5ae3fbc9..b49e7c2d 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -1,16 +1,16 @@ use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot}; use actix::{Actor, Addr}; use aggregator::{ - PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorRepositoryFactory, + PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorState, PublicKeyAggregator, PublicKeyAggregatorParams, - PublicKeyAggregatorRepositoryFactory, PublicKeyAggregatorState, + PublicKeyAggregatorState, }; use anyhow::{anyhow, Result}; use async_trait::async_trait; -use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; +use data::{ FromSnapshotWithParams, Snapshot}; use enclave_core::{BusError, E3Requested, EnclaveErrorType, EnclaveEvent, EventBus}; -use fhe::{Fhe, FheRepositoryFactory, SharedRng}; -use keyshare::{Keyshare, KeyshareParams, KeyshareRepositoryFactory}; +use fhe::{Fhe, SharedRng}; +use keyshare::{Keyshare, KeyshareParams}; use sortition::Sortition; use std::sync::Arc; @@ -42,7 +42,7 @@ impl E3Feature for FheFeature { // FHE doesn't implement Checkpoint so we are going to store it manually let fhe = Arc::new(Fhe::from_encoded(¶ms, seed, self.rng.clone()).unwrap()); - ctx.get_store().store().fhe(&e3_id).write(&fhe.snapshot()); + ctx.repositories().fhe(&e3_id).write(&fhe.snapshot()); let _ = ctx.set_fhe(fhe); } @@ -57,7 +57,7 @@ impl E3Feature for FheFeature { }; // No Snapshot returned from the store -> bail - let Some(snap) = ctx.get_store().store().fhe(&ctx.e3_id).read().await? else { + let Some(snap) = ctx.repositories().fhe(&ctx.e3_id).read().await? else { return Ok(()); }; @@ -100,7 +100,7 @@ impl E3Feature for KeyshareFeature { let _ = ctx.set_keyshare( Keyshare::new(KeyshareParams { bus: self.bus.clone(), - store: ctx.get_store().store().keyshare(&e3_id), + store: ctx.repositories().keyshare(&e3_id), fhe: fhe.clone(), address: self.address.clone(), }) @@ -118,7 +118,7 @@ impl E3Feature for KeyshareFeature { return Ok(()); }; - let store = ctx.get_store().store().keyshare(&snapshot.e3_id); + let store = ctx.repositories().keyshare(&snapshot.e3_id); // No Snapshot returned from the store -> bail let Some(snap) = store.read().await? else { @@ -185,7 +185,7 @@ impl E3Feature for PlaintextAggregatorFeature { PlaintextAggregatorParams { fhe: fhe.clone(), bus: self.bus.clone(), - store: ctx.get_store().store().plaintext(&e3_id), + store: ctx.repositories().plaintext(&e3_id), sortition: self.sortition.clone(), e3_id, src_chain_id: meta.src_chain_id, @@ -210,7 +210,7 @@ impl E3Feature for PlaintextAggregatorFeature { return Ok(()); } - let store = ctx.get_store().store().plaintext(&snapshot.e3_id); + let store = ctx.repositories().plaintext(&snapshot.e3_id); // No Snapshot returned from the store -> bail let Some(snap) = store.read().await? else { @@ -282,7 +282,7 @@ impl E3Feature for PublicKeyAggregatorFeature { PublicKeyAggregatorParams { fhe: fhe.clone(), bus: self.bus.clone(), - store: ctx.get_store().store().publickey(&e3_id), + store: ctx.repositories().publickey(&e3_id), sortition: self.sortition.clone(), e3_id, src_chain_id: meta.src_chain_id, @@ -303,7 +303,7 @@ impl E3Feature for PublicKeyAggregatorFeature { return Ok(()); }; - let repository = ctx.get_store().store().publickey(&ctx.e3_id); + let repository = ctx.repositories().publickey(&ctx.e3_id); // No Snapshot returned from the store -> bail let Some(snap) = repository.read().await? else { diff --git a/packages/ciphernode/router/src/lib.rs b/packages/ciphernode/router/src/lib.rs index bb58b8aa..77f7aa5c 100644 --- a/packages/ciphernode/router/src/lib.rs +++ b/packages/ciphernode/router/src/lib.rs @@ -1,17 +1,13 @@ mod ciphernode_selector; mod committee_meta; -mod committee_repository; mod context; -mod context_repository; mod e3_request_router; mod hooks; -mod router_repository; +mod repositories; pub use ciphernode_selector::*; pub use committee_meta::*; -pub use committee_repository::*; pub use context::*; -pub use context_repository::*; pub use e3_request_router::*; pub use hooks::*; -pub use router_repository::*; +pub use repositories::*; diff --git a/packages/ciphernode/router/src/repositories.rs b/packages/ciphernode/router/src/repositories.rs new file mode 100644 index 00000000..f2dcfc77 --- /dev/null +++ b/packages/ciphernode/router/src/repositories.rs @@ -0,0 +1,64 @@ +use crate::{CommitteeMeta, E3RequestContextSnapshot, E3RequestRouterSnapshot}; +use aggregator::{PlaintextAggregatorState, PublicKeyAggregatorState}; +use data::{DataStore, Repository}; +use enclave_core::E3id; +use fhe::FheSnapshot; +use keyshare::KeyshareState; +use sortition::SortitionModule; + +pub struct Repositories { + store: DataStore, +} + +impl From for Repositories { + fn from(value: DataStore) -> Self { + Repositories { store: value } + } +} + +impl Repositories { + fn new(store: DataStore) -> Self { + Repositories { store } + } +} + +impl From> for Repositories { + fn from(value: Repository) -> Self { + let store:DataStore = value.into(); + store.into() + } +} + +impl Repositories { + pub fn keyshare(&self, e3_id: &E3id) -> Repository { + Repository::new(self.store.scope(format!("//keyshare/{e3_id}"))) + } + + pub fn plaintext(&self, e3_id: &E3id) -> Repository { + Repository::new(self.store.scope(format!("//plaintext/{e3_id}"))) + } + + pub fn publickey(&self, e3_id: &E3id) -> Repository { + Repository::new(self.store.scope(format!("//publickey/{e3_id}"))) + } + + pub fn fhe(&self, e3_id: &E3id) -> Repository { + Repository::new(self.store.scope(format!("//fhe/{e3_id}"))) + } + + pub fn meta(&self, e3_id: &E3id) -> Repository { + Repository::new(self.store.scope(format!("//meta/{e3_id}"))) + } + + pub fn context(&self, e3_id: &E3id) -> Repository { + Repository::new(self.store.scope(format!("//context/{e3_id}"))) + } + + pub fn router(&self) -> Repository { + Repository::new(self.store.scope(format!("//router"))) + } + + pub fn sortition(&self) -> Repository { + Repository::new(self.store.scope(format!("//sortition"))) + } +} diff --git a/packages/ciphernode/router/src/router_repository.rs b/packages/ciphernode/router/src/router_repository.rs deleted file mode 100644 index 69fcfe67..00000000 --- a/packages/ciphernode/router/src/router_repository.rs +++ /dev/null @@ -1,29 +0,0 @@ -use async_trait::async_trait; -use data::{DataStore, Repository}; - -use crate::E3RequestRouterSnapshot; - -#[derive(Clone)] -pub struct E3RouterRepository { - store: DataStore, -} - -#[async_trait] -impl Repository for E3RouterRepository { - type State = E3RequestRouterSnapshot; - fn store(&self) -> DataStore { - self.store.clone() - } -} - -pub trait E3RouterRepositoryFactory { - fn router(&self) -> E3RouterRepository; -} - -impl E3RouterRepositoryFactory for DataStore { - fn router(&self) -> E3RouterRepository { - E3RouterRepository { - store: self.scope(format!("//router")), - } - } -} diff --git a/packages/ciphernode/sortition/src/lib.rs b/packages/ciphernode/sortition/src/lib.rs index 7113c06d..792f64fc 100644 --- a/packages/ciphernode/sortition/src/lib.rs +++ b/packages/ciphernode/sortition/src/lib.rs @@ -4,10 +4,8 @@ mod distance; mod index; -mod repository; mod sortition; pub use distance::*; pub use index::*; -pub use repository::*; pub use sortition::*; diff --git a/packages/ciphernode/sortition/src/repository.rs b/packages/ciphernode/sortition/src/repository.rs deleted file mode 100644 index e10c4149..00000000 --- a/packages/ciphernode/sortition/src/repository.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::SortitionModule; -use anyhow::*; -use async_trait::async_trait; -use data::{DataStore, Repository}; - -#[derive(Clone)] -pub struct SortitionRepository { - store: DataStore, -} - -#[async_trait] -impl Repository for SortitionRepository { - type State = SortitionModule; - async fn read(&self) -> Result> { - self.store.read().await - } - - fn write(&self, value: &SortitionModule) { - self.store.write(value) - } - - fn store(&self) -> DataStore { - self.store.clone() - } -} - -pub trait SortitionRepositoryFactory { - fn sortition(&self) -> SortitionRepository; -} - -impl SortitionRepositoryFactory for DataStore { - fn sortition(&self) -> SortitionRepository { - SortitionRepository { - store: self.scope(format!("//sortition")), - } - } -} diff --git a/packages/ciphernode/sortition/src/sortition.rs b/packages/ciphernode/sortition/src/sortition.rs index 97357c61..7794d088 100644 --- a/packages/ciphernode/sortition/src/sortition.rs +++ b/packages/ciphernode/sortition/src/sortition.rs @@ -1,9 +1,9 @@ -use crate::{DistanceSortition, SortitionRepository}; +use crate::DistanceSortition; use actix::prelude::*; use alloy::primitives::Address; use anyhow::{anyhow, Result}; use async_trait::async_trait; -use data::{Checkpoint, FromSnapshotWithParams, Snapshot}; +use data::{Checkpoint, FromSnapshotWithParams, Repository, Snapshot}; use enclave_core::{ BusError, CiphernodeAdded, CiphernodeRemoved, EnclaveErrorType, EnclaveEvent, EventBus, Seed, Subscribe, @@ -84,12 +84,12 @@ pub struct GetNodes; pub struct Sortition { list: SortitionModule, bus: Addr, - store: SortitionRepository, + store: Repository, } pub struct SortitionParams { pub bus: Addr, - pub store: SortitionRepository, + pub store: Repository, } impl Sortition { @@ -101,7 +101,7 @@ impl Sortition { } } - pub fn attach(bus: Addr, store: SortitionRepository) -> Addr { + pub fn attach(bus: Addr, store: Repository) -> Addr { let addr = Sortition::new(SortitionParams { bus: bus.clone(), store, @@ -140,8 +140,7 @@ impl FromSnapshotWithParams for Sortition { } impl Checkpoint for Sortition { - type Repository = SortitionRepository; - fn get_store(&self) -> SortitionRepository { + fn repository(&self) -> Repository { self.store.clone() } } diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 88493815..927bb218 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -9,7 +9,7 @@ use logger::SimpleLogger; use p2p::P2p; use router::{ CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, PlaintextAggregatorFeature, - PublicKeyAggregatorFeature, + PublicKeyAggregatorFeature, Repositories, }; use sortition::Sortition; @@ -38,9 +38,10 @@ async fn setup_local_ciphernode( // create data actor for saving data let data_actor = InMemStore::new(logging).start(); // TODO: Use a sled backed Data Actor let store = DataStore::from_in_mem(data_actor); + let repositories: Repositories = store.clone().into(); // create ciphernode actor for managing ciphernode flow - let sortition = Sortition::attach(bus.clone(), store.clone()); + let sortition = Sortition::attach(bus.clone(), repositories.sortition()); CiphernodeSelector::attach(bus.clone(), sortition.clone(), addr); E3RequestRouter::builder(bus.clone(), store) From ebd953e882b468773bc05e8489fae440a98bd9bf Mon Sep 17 00:00:00 2001 From: ktdlr Date: Fri, 18 Oct 2024 19:47:28 +1100 Subject: [PATCH 17/49] Formatting and make constructor public --- packages/ciphernode/aggregator/src/publickey_aggregator.rs | 1 - packages/ciphernode/data/src/repository.rs | 2 +- packages/ciphernode/keyshare/src/keyshare.rs | 2 +- packages/ciphernode/router/src/hooks.rs | 7 +++---- packages/ciphernode/router/src/repositories.rs | 4 ++-- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/ciphernode/aggregator/src/publickey_aggregator.rs b/packages/ciphernode/aggregator/src/publickey_aggregator.rs index 81d98910..8f19c21f 100644 --- a/packages/ciphernode/aggregator/src/publickey_aggregator.rs +++ b/packages/ciphernode/aggregator/src/publickey_aggregator.rs @@ -10,7 +10,6 @@ use sortition::{GetHasNode, GetNodes, Sortition}; use std::sync::Arc; use tracing::error; - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum PublicKeyAggregatorState { Collecting { diff --git a/packages/ciphernode/data/src/repository.rs b/packages/ciphernode/data/src/repository.rs index c54de5bd..e1c6d803 100644 --- a/packages/ciphernode/data/src/repository.rs +++ b/packages/ciphernode/data/src/repository.rs @@ -18,7 +18,7 @@ impl Repository { } } -impl Deref for Repository{ +impl Deref for Repository { type Target = DataStore; fn deref(&self) -> &Self::Target { &self.store diff --git a/packages/ciphernode/keyshare/src/keyshare.rs b/packages/ciphernode/keyshare/src/keyshare.rs index 5f29a3b5..f30c89d3 100644 --- a/packages/ciphernode/keyshare/src/keyshare.rs +++ b/packages/ciphernode/keyshare/src/keyshare.rs @@ -57,7 +57,7 @@ impl Snapshot for Keyshare { } impl Checkpoint for Keyshare { - fn repository(&self) -> Repository{ + fn repository(&self) -> Repository { self.store.clone() } } diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index b49e7c2d..f8c02b46 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -1,13 +1,12 @@ use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot}; use actix::{Actor, Addr}; use aggregator::{ - PlaintextAggregator, PlaintextAggregatorParams, - PlaintextAggregatorState, PublicKeyAggregator, PublicKeyAggregatorParams, - PublicKeyAggregatorState, + PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorState, PublicKeyAggregator, + PublicKeyAggregatorParams, PublicKeyAggregatorState, }; use anyhow::{anyhow, Result}; use async_trait::async_trait; -use data::{ FromSnapshotWithParams, Snapshot}; +use data::{FromSnapshotWithParams, Snapshot}; use enclave_core::{BusError, E3Requested, EnclaveErrorType, EnclaveEvent, EventBus}; use fhe::{Fhe, SharedRng}; use keyshare::{Keyshare, KeyshareParams}; diff --git a/packages/ciphernode/router/src/repositories.rs b/packages/ciphernode/router/src/repositories.rs index f2dcfc77..8a5f9d60 100644 --- a/packages/ciphernode/router/src/repositories.rs +++ b/packages/ciphernode/router/src/repositories.rs @@ -17,14 +17,14 @@ impl From for Repositories { } impl Repositories { - fn new(store: DataStore) -> Self { + pub fn new(store: DataStore) -> Self { Repositories { store } } } impl From> for Repositories { fn from(value: Repository) -> Self { - let store:DataStore = value.into(); + let store: DataStore = value.into(); store.into() } } From a82c528b99f13934116b0325f658d1105502a93b Mon Sep 17 00:00:00 2001 From: ktdlr Date: Fri, 18 Oct 2024 20:04:01 +1100 Subject: [PATCH 18/49] Enable easy access to repositories --- .../ciphernode/enclave_node/src/aggregator.rs | 4 ++-- .../ciphernode/enclave_node/src/ciphernode.rs | 4 ++-- .../ciphernode/router/src/committee_meta.rs | 2 +- packages/ciphernode/router/src/context.rs | 12 ++++++---- .../router/src/e3_request_router.rs | 10 ++++---- packages/ciphernode/router/src/hooks.rs | 2 +- .../ciphernode/router/src/repositories.rs | 24 +++++++++++++++++++ .../tests/test_aggregation_and_decryption.rs | 4 ++-- 8 files changed, 44 insertions(+), 18 deletions(-) diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index c0270ab4..d3307b78 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -11,7 +11,7 @@ use rand::SeedableRng; use rand_chacha::rand_core::OsRng; use router::{ E3RequestRouter, FheFeature, PlaintextAggregatorFeature, PublicKeyAggregatorFeature, - Repositories, + RepositoriesFactory, }; use sortition::Sortition; use std::sync::{Arc, Mutex}; @@ -56,7 +56,7 @@ impl MainAggregator { )); let store = DataStore::from_in_mem(InMemStore::new(true).start()); - let repositories: Repositories = store.clone().into(); + let repositories = store.repositories(); let sortition = Sortition::attach(bus.clone(), repositories.sortition()); let signer = pull_eth_signer_from_env("PRIVATE_KEY").await?; for chain in config diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 0671f855..00e8b28b 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -8,7 +8,7 @@ use logger::SimpleLogger; use p2p::P2p; use rand::SeedableRng; use rand_chacha::rand_core::OsRng; -use router::{CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, Repositories}; +use router::{CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, Repositories, RepositoriesFactory}; use sortition::Sortition; use std::sync::{Arc, Mutex}; use tokio::task::JoinHandle; @@ -59,7 +59,7 @@ impl MainCiphernode { let bus = EventBus::new(true).start(); // TODO: switch to Sled actor let store = DataStore::from_in_mem(InMemStore::new(true).start()); - let repositories: Repositories = store.clone().into(); + let repositories = store.repositories(); let sortition = Sortition::attach(bus.clone(), repositories.sortition()); let selector = diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index 0749f040..4beed2c8 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -1,4 +1,4 @@ -use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot}; +use crate::{E3Feature, E3RequestContext, RepositoriesFactory, E3RequestContextSnapshot}; use anyhow::*; use async_trait::async_trait; use enclave_core::{E3Requested, EnclaveEvent, Seed}; diff --git a/packages/ciphernode/router/src/context.rs b/packages/ciphernode/router/src/context.rs index 81bcf4ca..22d2e4d1 100644 --- a/packages/ciphernode/router/src/context.rs +++ b/packages/ciphernode/router/src/context.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use crate::{CommitteeMeta, E3Feature, EventBuffer, Repositories}; +use crate::{CommitteeMeta, E3Feature, EventBuffer, Repositories, RepositoriesFactory}; use actix::{Addr, Recipient}; use aggregator::{PlaintextAggregator, PublicKeyAggregator}; use anyhow::Result; @@ -82,10 +82,6 @@ impl E3RequestContext { }); } - pub fn repositories(&self) -> Repositories { - self.repository().into() - } - /// Accept a DataStore ID and a Keystore actor address pub fn set_keyshare(&mut self, value: Addr) { self.keyshare = Some(value); @@ -137,6 +133,12 @@ impl E3RequestContext { } } +impl RepositoriesFactory for E3RequestContext { + fn repositories(&self) -> Repositories { + self.repository().into() + } +} + #[async_trait] impl Snapshot for E3RequestContext { type Snapshot = E3RequestContextSnapshot; diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index c4554a51..daf57787 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -2,7 +2,7 @@ use crate::CommitteeMetaFeature; use crate::E3RequestContext; use crate::E3RequestContextParams; use crate::E3RequestContextSnapshot; -use crate::Repositories; +use crate::RepositoriesFactory; use actix::{Actor, Addr, Context, Handler}; use anyhow::*; use async_trait::async_trait; @@ -74,7 +74,7 @@ pub struct E3RequestRouterParams { impl E3RequestRouter { pub fn builder(bus: Addr, store: DataStore) -> E3RequestRouterBuilder { - let repositories: Repositories = store.into(); + let repositories = store.repositories(); let builder = E3RequestRouterBuilder { bus, features: vec![], @@ -115,7 +115,7 @@ impl Handler for E3RequestRouter { return; } - let repositories: Repositories = self.repository().into(); + let repositories = self.repository().repositories(); let context = self.contexts.entry(e3_id.clone()).or_insert_with(|| { E3RequestContext::from_params(E3RequestContextParams { e3_id: e3_id.clone(), @@ -187,7 +187,7 @@ impl FromSnapshotWithParams for E3RequestRouter { async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { let mut contexts = HashMap::new(); - let repositories: Repositories = params.store.into(); + let repositories = params.store.repositories(); for e3_id in snapshot.contexts { let Some(ctx_snapshot) = repositories.context(&e3_id).read().await? else { continue; @@ -232,7 +232,7 @@ impl E3RequestRouterBuilder { } pub async fn build(self) -> Result> { - let repositories: Repositories = self.store.into(); + let repositories = self.store.repositories(); let router_repo = repositories.router(); let snapshot: Option = router_repo.read().await?; let params = E3RequestRouterParams { diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index f8c02b46..7fbcc831 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -1,4 +1,4 @@ -use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot}; +use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot, RepositoriesFactory}; use actix::{Actor, Addr}; use aggregator::{ PlaintextAggregator, PlaintextAggregatorParams, PlaintextAggregatorState, PublicKeyAggregator, diff --git a/packages/ciphernode/router/src/repositories.rs b/packages/ciphernode/router/src/repositories.rs index 8a5f9d60..831587e4 100644 --- a/packages/ciphernode/router/src/repositories.rs +++ b/packages/ciphernode/router/src/repositories.rs @@ -15,6 +15,13 @@ impl From for Repositories { Repositories { store: value } } } +impl From<&DataStore> for Repositories { + fn from(value: &DataStore) -> Self { + Repositories { + store: value.clone(), + } + } +} impl Repositories { pub fn new(store: DataStore) -> Self { @@ -62,3 +69,20 @@ impl Repositories { Repository::new(self.store.scope(format!("//sortition"))) } } + +pub trait RepositoriesFactory { + fn repositories(&self) -> Repositories; +} + +impl RepositoriesFactory for DataStore { + fn repositories(&self) -> Repositories { + self.into() + } +} + +impl RepositoriesFactory for Repository { + fn repositories(&self) -> Repositories { + let store:DataStore = self.clone().into(); + store.repositories() + } +} diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 927bb218..30e8c7f0 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -9,7 +9,7 @@ use logger::SimpleLogger; use p2p::P2p; use router::{ CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, PlaintextAggregatorFeature, - PublicKeyAggregatorFeature, Repositories, + PublicKeyAggregatorFeature, Repositories, RepositoriesFactory, }; use sortition::Sortition; @@ -38,7 +38,7 @@ async fn setup_local_ciphernode( // create data actor for saving data let data_actor = InMemStore::new(logging).start(); // TODO: Use a sled backed Data Actor let store = DataStore::from_in_mem(data_actor); - let repositories: Repositories = store.clone().into(); + let repositories = store.repositories(); // create ciphernode actor for managing ciphernode flow let sortition = Sortition::attach(bus.clone(), repositories.sortition()); From f4e54a27cd863441e36f1e6e52eaedc4de8453e7 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Fri, 18 Oct 2024 20:04:37 +1100 Subject: [PATCH 19/49] Formatting --- packages/ciphernode/enclave_node/src/ciphernode.rs | 5 ++++- packages/ciphernode/router/src/committee_meta.rs | 2 +- packages/ciphernode/router/src/repositories.rs | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 00e8b28b..bac04dde 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -8,7 +8,10 @@ use logger::SimpleLogger; use p2p::P2p; use rand::SeedableRng; use rand_chacha::rand_core::OsRng; -use router::{CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, Repositories, RepositoriesFactory}; +use router::{ + CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, Repositories, + RepositoriesFactory, +}; use sortition::Sortition; use std::sync::{Arc, Mutex}; use tokio::task::JoinHandle; diff --git a/packages/ciphernode/router/src/committee_meta.rs b/packages/ciphernode/router/src/committee_meta.rs index 4beed2c8..eb9a7d2c 100644 --- a/packages/ciphernode/router/src/committee_meta.rs +++ b/packages/ciphernode/router/src/committee_meta.rs @@ -1,4 +1,4 @@ -use crate::{E3Feature, E3RequestContext, RepositoriesFactory, E3RequestContextSnapshot}; +use crate::{E3Feature, E3RequestContext, E3RequestContextSnapshot, RepositoriesFactory}; use anyhow::*; use async_trait::async_trait; use enclave_core::{E3Requested, EnclaveEvent, Seed}; diff --git a/packages/ciphernode/router/src/repositories.rs b/packages/ciphernode/router/src/repositories.rs index 831587e4..1d93c179 100644 --- a/packages/ciphernode/router/src/repositories.rs +++ b/packages/ciphernode/router/src/repositories.rs @@ -82,7 +82,7 @@ impl RepositoriesFactory for DataStore { impl RepositoriesFactory for Repository { fn repositories(&self) -> Repositories { - let store:DataStore = self.clone().into(); + let store: DataStore = self.clone().into(); store.repositories() } } From 89684be67b24a0a79a71bc92003de41a57373490 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Mon, 21 Oct 2024 14:30:44 +1100 Subject: [PATCH 20/49] Refactor test case to add new test --- .../tests/test_aggregation_and_decryption.rs | 402 +++++++++++------- 1 file changed, 248 insertions(+), 154 deletions(-) diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 30e8c7f0..4c92d461 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -9,7 +9,7 @@ use logger::SimpleLogger; use p2p::P2p; use router::{ CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, PlaintextAggregatorFeature, - PublicKeyAggregatorFeature, Repositories, RepositoriesFactory, + PublicKeyAggregatorFeature, RepositoriesFactory, }; use sortition::Sortition; @@ -17,7 +17,7 @@ use actix::prelude::*; use alloy::primitives::Address; use anyhow::*; use fhe_rs::{ - bfv::{BfvParameters, Encoding, Plaintext, PublicKey, SecretKey}, + bfv::{BfvParameters, Ciphertext, Encoding, Plaintext, PublicKey, SecretKey}, mbfv::{AggregateIter, CommonRandomPoly, DecryptionShare, PublicKeyShare}, }; use fhe_traits::{FheEncoder, FheEncrypter, Serialize}; @@ -30,8 +30,8 @@ use tokio::{sync::mpsc::channel, time::sleep}; // Simulating a local node async fn setup_local_ciphernode( - bus: Addr, - rng: SharedRng, + bus: &Addr, + rng: &SharedRng, logging: bool, addr: &str, ) -> Result<()> { @@ -45,7 +45,7 @@ async fn setup_local_ciphernode( CiphernodeSelector::attach(bus.clone(), sortition.clone(), addr); E3RequestRouter::builder(bus.clone(), store) - .add_feature(FheFeature::create(rng)) + .add_feature(FheFeature::create(rng.clone())) .add_feature(PublicKeyAggregatorFeature::create( bus.clone(), sortition.clone(), @@ -63,209 +63,300 @@ async fn setup_local_ciphernode( } fn generate_pk_share( - params: Arc, - crp: CommonRandomPoly, - rng: SharedRng, -) -> Result<(PublicKeyShare, SecretKey)> { + params: &Arc, + crp: &CommonRandomPoly, + rng: &SharedRng, + addr: &str, +) -> Result { let sk = SecretKey::random(¶ms, &mut *rng.lock().unwrap()); let pk = PublicKeyShare::new(&sk, crp.clone(), &mut *rng.lock().unwrap())?; - Ok((pk, sk)) + Ok((pk, sk, addr.to_owned())) } -#[actix::test] -async fn test_public_key_aggregation_and_decryption() -> Result<()> { - // Setup EventBus - let bus = EventBus::new(true).start(); - let rng = Arc::new(std::sync::Mutex::new(ChaCha20Rng::seed_from_u64(42))); - let seed = Seed(ChaCha20Rng::seed_from_u64(123).get_seed()); +fn generate_pk_shares( + params: &Arc, + crp: &CommonRandomPoly, + rng: &SharedRng, + eth_addrs: &Vec, +) -> Result> { + let mut result = vec![]; + for addr in eth_addrs { + result.push(generate_pk_share(params, crp, rng, addr)?); + } + Ok(result) +} - let eth_addrs: Vec = (0..3) +fn create_random_eth_addrs(how_many: u32) -> Vec { + (0..how_many) .map(|_| Address::from_slice(&rand::thread_rng().gen::<[u8; 20]>()).to_string()) - .collect(); + .collect() +} - setup_local_ciphernode(bus.clone(), rng.clone(), true, ð_addrs[0]).await?; - setup_local_ciphernode(bus.clone(), rng.clone(), true, ð_addrs[1]).await?; - setup_local_ciphernode(bus.clone(), rng.clone(), true, ð_addrs[2]).await?; +fn create_shared_rng_from_u64(value: u64) -> Arc> { + Arc::new(std::sync::Mutex::new(ChaCha20Rng::seed_from_u64(value))) +} - let e3_id = E3id::new("1234"); +fn create_seed_from_u64(value: u64) -> Seed { + Seed(ChaCha20Rng::seed_from_u64(value).get_seed()) +} +fn create_crp_bytes_params( + moduli: &[u64], + degree: usize, + plaintext_modulus: u64, + seed: &Seed, +) -> (Vec, Arc) { let ParamsWithCrp { crp_bytes, params, .. } = setup_crp_params( - &[0x3FFFFFFF000001], - 2048, - 1032193, + moduli, + degree, + plaintext_modulus, Arc::new(std::sync::Mutex::new(ChaCha20Rng::from_seed( seed.clone().into(), ))), ); + (crp_bytes, params) +} - let regevt_1 = EnclaveEvent::from(CiphernodeAdded { - address: eth_addrs[0].clone(), - index: 0, - num_nodes: 1, - }); +/// Test helper to add addresses to the committee by creating events on the event bus +struct AddToCommittee { + bus: Addr, + count: usize, +} - bus.send(regevt_1.clone()).await?; +impl AddToCommittee { + fn new(bus: &Addr) -> Self { + Self { + bus: bus.clone(), + count: 0, + } + } + async fn add(&mut self, address: &str) -> Result { + let evt = EnclaveEvent::from(CiphernodeAdded { + address: address.to_owned(), + index: self.count, + num_nodes: self.count + 1, + }); - let regevt_2 = EnclaveEvent::from(CiphernodeAdded { - address: eth_addrs[1].clone(), - index: 1, - num_nodes: 2, - }); + self.count += 1; - bus.send(regevt_2.clone()).await?; + self.bus.send(evt.clone()).await?; - let regevt_3 = EnclaveEvent::from(CiphernodeAdded { - address: eth_addrs[2].clone(), - index: 2, - num_nodes: 3, - }); + Ok(evt) + } +} - bus.send(regevt_3.clone()).await?; +async fn create_local_ciphernodes( + bus: &Addr, + rng: &SharedRng, + count: u32, +) -> Result> { + let eth_addrs = create_random_eth_addrs(count); - let event = EnclaveEvent::from(E3Requested { + for addr in ð_addrs { + setup_local_ciphernode(&bus, &rng, true, addr).await?; + } + + Ok(eth_addrs) +} + +fn encrypt_ciphertext( + params: &Arc, + pubkey: PublicKey, + raw_plaintext: Vec, +) -> Result<(Arc, Vec)> { + let padded = &pad_end(&raw_plaintext, 0, 2048); + let expected = bincode::serialize(&padded)?; + let pt = Plaintext::try_encode(&raw_plaintext, Encoding::poly(), ¶ms)?; + let ciphertext = pubkey.try_encrypt(&pt, &mut ChaCha20Rng::seed_from_u64(42))?; + Ok((Arc::new(ciphertext), expected)) +} + +fn pad_end(input: &[u64], pad: u64, total: usize) -> Vec { + let len = input.len(); + let mut cop = input.to_vec(); + cop.extend(std::iter::repeat(pad).take(total - len)); + cop +} + +async fn add_ciphernodes(bus: &Addr, addrs: &Vec) -> Result> { + let mut committee = AddToCommittee::new(&bus); + let mut evts: Vec = vec![]; + + for addr in addrs { + evts.push(committee.add(addr).await?); + } + Ok(evts) +} + +// Type for our tests to test against +type PkSkShareTuple = (PublicKeyShare, SecretKey, String); +type DecryptionShareTuple = (Vec, String); + +fn aggregate_public_key(shares: &Vec) -> Result { + Ok(shares + .clone() + .into_iter() + .map(|(pk, _, _)| pk) + .aggregate()?) +} + +fn to_decryption_shares( + shares: &Vec, + ciphertext: &Arc, + rng: &SharedRng, +) -> Result> { + let mut results = vec![]; + for (_, sk, addr) in shares { + results.push(( + DecryptionShare::new(&sk, &ciphertext, &mut *rng.lock().unwrap())?.to_bytes(), + addr.to_owned(), + )); + } + + Ok(results) +} + +/// Helper to create keyshare events from eth addresses and generated shares +fn to_keyshare_events(shares: &Vec, e3_id: &E3id) -> Vec { + let mut result = Vec::new(); + for i in 0..shares.len() { + result.push(EnclaveEvent::from(KeyshareCreated { + pubkey: shares[i].0.to_bytes(), + e3_id: e3_id.clone(), + node: shares[i].2.clone(), + })); + } + result +} + +fn to_decryptionshare_events( + decryption_shares: &Vec, + e3_id: &E3id, +) -> Vec { + let mut result = Vec::new(); + for i in 0..decryption_shares.len() { + result.push(EnclaveEvent::from(DecryptionshareCreated { + decryption_share: decryption_shares[i].0.clone(), + e3_id: e3_id.clone(), + node: decryption_shares[i].1.clone(), + })); + } + result +} + +#[actix::test] +async fn test_public_key_aggregation_and_decryption() -> Result<()> { + // Setup + let bus = EventBus::new(true).start(); + let rng = create_shared_rng_from_u64(42); + let seed = create_seed_from_u64(123); + let (crp_bytes, params) = create_crp_bytes_params(&[0x3FFFFFFF000001], 2048, 1032193, &seed); + let crpoly = CommonRandomPoly::deserialize(&crp_bytes.clone(), ¶ms)?; + let e3_id = E3id::new("1234"); + + + // Setup actual ciphernodes and dispatch add events + let eth_addrs = create_local_ciphernodes(&bus, &rng, 3).await?; + let add_events = add_ciphernodes(&bus, ð_addrs).await?; + let e3_request_event = EnclaveEvent::from(E3Requested { e3_id: e3_id.clone(), threshold_m: 3, seed: seed.clone(), params: params.to_bytes(), src_chain_id: 1, }); + // Send the computation requested event - bus.send(event.clone()).await?; + bus.send(e3_request_event.clone()).await?; // Test that we cannot send the same event twice - bus.send(event).await?; + bus.send(e3_request_event.clone()).await?; + + // Generate the test shares and pubkey + let rng_test = create_shared_rng_from_u64(42); + let test_shares = generate_pk_shares(¶ms, &crpoly, &rng_test, ð_addrs)?; + let test_pubkey = aggregate_public_key(&test_shares)?; + + // Assemble the expected history + // Rust doesn't have a spread operator so this is a little awkward + let mut expected_history = vec![]; + expected_history.extend(add_events); // start with add events + expected_history.extend(vec![ + // The e3 request + e3_request_event, + // Ciphernode is selected + EnclaveEvent::from(CiphernodeSelected { + e3_id: e3_id.clone(), + threshold_m: 3, + }), + ]); + // Keyshare events + expected_history.extend(to_keyshare_events(&test_shares, &e3_id)); + expected_history.extend(vec![ + // Our key has been aggregated + EnclaveEvent::from(PublicKeyAggregated { + pubkey: test_pubkey.to_bytes(), + e3_id: e3_id.clone(), + nodes: OrderedSet::from(eth_addrs.clone()), + src_chain_id: 1, + }), + ]); let history = bus.send(GetHistory).await?; - - let rng_test = Arc::new(std::sync::Mutex::new(ChaCha20Rng::seed_from_u64(42))); - - let crpoly = CommonRandomPoly::deserialize(&crp_bytes.clone(), ¶ms)?; - - let (p1, sk1) = generate_pk_share(params.clone(), crpoly.clone(), rng_test.clone())?; - let (p2, sk2) = generate_pk_share(params.clone(), crpoly.clone(), rng_test.clone())?; - let (p3, sk3) = generate_pk_share(params.clone(), crpoly.clone(), rng_test.clone())?; - - let pubkey: PublicKey = vec![p1.clone(), p2.clone(), p3.clone()] - .into_iter() - .aggregate()?; - assert_eq!(history.len(), 9); - assert_eq!( - history, - vec![ - regevt_1, - regevt_2, - regevt_3, - EnclaveEvent::from(E3Requested { - e3_id: e3_id.clone(), - threshold_m: 3, - seed: seed.clone(), - params: params.to_bytes(), - src_chain_id: 1 - }), - EnclaveEvent::from(CiphernodeSelected { - e3_id: e3_id.clone(), - threshold_m: 3, - }), - EnclaveEvent::from(KeyshareCreated { - pubkey: p1.to_bytes(), - e3_id: e3_id.clone(), - node: eth_addrs[0].clone() - }), - EnclaveEvent::from(KeyshareCreated { - pubkey: p2.to_bytes(), - e3_id: e3_id.clone(), - node: eth_addrs[1].clone() - }), - EnclaveEvent::from(KeyshareCreated { - pubkey: p3.to_bytes(), - e3_id: e3_id.clone(), - node: eth_addrs[2].clone() - }), - EnclaveEvent::from(PublicKeyAggregated { - pubkey: pubkey.to_bytes(), - e3_id: e3_id.clone(), - nodes: OrderedSet::from(eth_addrs.clone()), - src_chain_id: 1 - }), - ] - ); + assert_eq!(history, expected_history); + bus.send(ResetHistory).await?; // Aggregate decryption - bus.send(ResetHistory).await?; - fn pad_end(input: &[u64], pad: u64, total: usize) -> Vec { - let len = input.len(); - let mut cop = input.to_vec(); - cop.extend(std::iter::repeat(pad).take(total - len)); - cop - } + // TODO: // Making these values large (especially the yes value) requires changing // the params we use here - as we tune the FHE we need to take care - let yes = 1234u64; - let no = 873827u64; - - let raw_plaintext = vec![yes, no]; - let padded = &pad_end(&raw_plaintext, 0, 2048); - let expected_raw_plaintext = bincode::serialize(&padded)?; - let pt = Plaintext::try_encode(&raw_plaintext, Encoding::poly(), ¶ms)?; - - let ciphertext = pubkey.try_encrypt(&pt, &mut ChaCha20Rng::seed_from_u64(42))?; + let raw_plaintext = vec![1234u64, 873827u64]; + let (ciphertext, expected) = encrypt_ciphertext(¶ms, test_pubkey, raw_plaintext)?; + let decryption_events = to_decryptionshare_events( + &to_decryption_shares(&test_shares, &ciphertext, &rng_test)?, + &e3_id, + ); - let event = EnclaveEvent::from(CiphertextOutputPublished { + // Setup Ciphertext Published Event + let ciphertext_published_event = EnclaveEvent::from(CiphertextOutputPublished { ciphertext_output: ciphertext.to_bytes(), e3_id: e3_id.clone(), }); - let arc_ct = Arc::new(ciphertext); + bus.send(ciphertext_published_event.clone()).await?; - let ds1 = DecryptionShare::new(&sk1, &arc_ct, &mut *rng_test.lock().unwrap())?.to_bytes(); - let ds2 = DecryptionShare::new(&sk2, &arc_ct, &mut *rng_test.lock().unwrap())?.to_bytes(); - let ds3 = DecryptionShare::new(&sk3, &arc_ct, &mut *rng_test.lock().unwrap())?.to_bytes(); + sleep(Duration::from_millis(1)).await; // need to push to next tick - // let ds1 = sk1 - bus.send(event.clone()).await?; + // Assemble the expected history + // Rust doesn't have a spread operator so this is a little awkward + let mut expected_history = vec![]; + expected_history.extend(vec![ciphertext_published_event.clone()]); + expected_history.extend(decryption_events); + expected_history.extend(vec![ + EnclaveEvent::from(PlaintextAggregated { + e3_id: e3_id.clone(), + decrypted_output: expected.clone(), + src_chain_id: 1, + }), + EnclaveEvent::from(E3RequestComplete { + e3_id: e3_id.clone(), + }), + ]); - sleep(Duration::from_millis(1)).await; // need to push to next tick let history = bus.send(GetHistory).await?; - assert_eq!(history.len(), 6); - - assert_eq!( - history, - vec![ - event.clone(), - EnclaveEvent::from(DecryptionshareCreated { - decryption_share: ds1.clone(), - e3_id: e3_id.clone(), - node: eth_addrs[0].clone() - }), - EnclaveEvent::from(DecryptionshareCreated { - decryption_share: ds2.clone(), - e3_id: e3_id.clone(), - node: eth_addrs[1].clone() - }), - EnclaveEvent::from(DecryptionshareCreated { - decryption_share: ds3.clone(), - e3_id: e3_id.clone(), - node: eth_addrs[2].clone() - }), - EnclaveEvent::from(PlaintextAggregated { - e3_id: e3_id.clone(), - decrypted_output: expected_raw_plaintext.clone(), - src_chain_id: 1 - }), - EnclaveEvent::from(E3RequestComplete { - e3_id: e3_id.clone() - }) - ] - ); + assert_eq!(history, expected_history); Ok(()) } +#[actix::test] +async fn test_actors_all_die_when_requested() {} + #[actix::test] async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { // Setup elements in test @@ -331,6 +422,9 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { Ok(()) } +// #[actix::test] +// async fn test_actors_resume_from_shutdown() -> Result<()> {} + #[actix::test] async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { let seed = Seed(ChaCha20Rng::seed_from_u64(123).get_seed()); From 4beebe7a2ec30220be602bedf26fff3c161c4880 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Mon, 21 Oct 2024 20:45:06 +1100 Subject: [PATCH 21/49] Add test for hydration --- packages/ciphernode/Cargo.lock | 1 + packages/ciphernode/core/src/events.rs | 36 ++++- packages/ciphernode/data/src/data_store.rs | 4 +- .../ciphernode/enclave_node/src/aggregator.rs | 2 +- .../ciphernode/enclave_node/src/ciphernode.rs | 2 +- packages/ciphernode/keyshare/Cargo.toml | 1 + packages/ciphernode/keyshare/src/keyshare.rs | 3 + .../router/src/ciphernode_selector.rs | 28 +++- packages/ciphernode/router/src/context.rs | 8 + .../router/src/e3_request_router.rs | 20 ++- .../tests/test_aggregation_and_decryption.rs | 139 +++++++++++++++--- 11 files changed, 212 insertions(+), 32 deletions(-) diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 2a795236..efc250b2 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -3194,6 +3194,7 @@ dependencies = [ "enclave-core", "fhe 0.1.0", "serde", + "tracing", ] [[package]] diff --git a/packages/ciphernode/core/src/events.rs b/packages/ciphernode/core/src/events.rs index 527ad643..925028f3 100644 --- a/packages/ciphernode/core/src/events.rs +++ b/packages/ciphernode/core/src/events.rs @@ -120,10 +120,13 @@ pub enum EnclaveEvent { id: EventId, data: E3RequestComplete, }, - // CommitteeSelected, - // OutputDecrypted, - // CiphernodeRegistered, - // CiphernodeDeregistered, + Shutdown { + id: EventId, + data: Shutdown, + }, // CommitteeSelected, + // OutputDecrypted, + // CiphernodeRegistered, + // CiphernodeDeregistered, } impl EnclaveEvent { @@ -147,6 +150,7 @@ impl EnclaveEvent { EnclaveEvent::CiphernodeAdded { .. } => true, EnclaveEvent::CiphernodeRemoved { .. } => true, EnclaveEvent::E3RequestComplete { .. } => true, + EnclaveEvent::Shutdown { .. } => true, _ => false, } } @@ -166,6 +170,7 @@ impl From for EventId { EnclaveEvent::CiphernodeRemoved { id, .. } => id, EnclaveEvent::EnclaveError { id, .. } => id, EnclaveEvent::E3RequestComplete { id, .. } => id, + EnclaveEvent::Shutdown { id, .. } => id, } } } @@ -196,6 +201,7 @@ impl EnclaveEvent { EnclaveEvent::CiphernodeRemoved { data, .. } => format!("{}", data), EnclaveEvent::E3RequestComplete { data, .. } => format!("{}", data), EnclaveEvent::EnclaveError { data, .. } => format!("{:?}", data), + EnclaveEvent::Shutdown { data, .. } => format!("{:?}", data), // _ => "".to_string(), } } @@ -206,6 +212,8 @@ pub trait FromError { fn from_error(err_type: EnclaveErrorType, error: Self::Error) -> Self; } + +// TODO: These From traits should be handled by a macro impl From for EnclaveEvent { fn from(data: KeyshareCreated) -> Self { EnclaveEvent::KeyshareCreated { @@ -305,6 +313,15 @@ impl From for EnclaveEvent { } } +impl From for EnclaveEvent { + fn from(data: Shutdown) -> Self { + EnclaveEvent::Shutdown { + id: EventId::from(data.clone()), + data: data.clone(), + } + } +} + impl FromError for EnclaveEvent { type Error = anyhow::Error; fn from_error(err_type: EnclaveErrorType, error: Self::Error) -> Self { @@ -505,6 +522,17 @@ impl Display for Die { } } + +/// Represents a shutdown event triggered by SIG TERM +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[rtype(result = "()")] +pub struct Shutdown; +impl Display for Shutdown { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Shutdown",) + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct Seed(pub [u8; 32]); impl From for u64 { diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index a8c8f1d6..4ba59060 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -64,7 +64,7 @@ impl DataStore { } /// Construct a data store from an InMemStore actor - pub fn from_in_mem(addr: Addr) -> Self { + pub fn from_in_mem(addr: &Addr) -> Self { Self { get: addr.clone().recipient(), insert: addr.clone().recipient(), @@ -88,7 +88,7 @@ impl DataStore { /// #[actix_rt::main] /// async fn main() -> Result<()>{ /// let addr = InMemStore::new(false).start(); - /// let store = DataStore::from_in_mem(addr); + /// let store = DataStore::from_in_mem(&addr); /// assert_eq!(store.base("//foo") /// .scope("bar") /// .scope("/baz") diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index d3307b78..328ed40d 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -55,7 +55,7 @@ impl MainAggregator { rand_chacha::ChaCha20Rng::from_rng(OsRng).expect("Failed to create RNG"), )); - let store = DataStore::from_in_mem(InMemStore::new(true).start()); + let store = DataStore::from_in_mem(&InMemStore::new(true).start()); let repositories = store.repositories(); let sortition = Sortition::attach(bus.clone(), repositories.sortition()); let signer = pull_eth_signer_from_env("PRIVATE_KEY").await?; diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index bac04dde..2f74f37b 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -61,7 +61,7 @@ impl MainCiphernode { )); let bus = EventBus::new(true).start(); // TODO: switch to Sled actor - let store = DataStore::from_in_mem(InMemStore::new(true).start()); + let store = DataStore::from_in_mem(&InMemStore::new(true).start()); let repositories = store.repositories(); let sortition = Sortition::attach(bus.clone(), repositories.sortition()); diff --git a/packages/ciphernode/keyshare/Cargo.toml b/packages/ciphernode/keyshare/Cargo.toml index b25e6bad..f4084acd 100644 --- a/packages/ciphernode/keyshare/Cargo.toml +++ b/packages/ciphernode/keyshare/Cargo.toml @@ -11,3 +11,4 @@ actix = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } async-trait = { workspace = true } +tracing = { workspace = true } diff --git a/packages/ciphernode/keyshare/src/keyshare.rs b/packages/ciphernode/keyshare/src/keyshare.rs index f30c89d3..bca9c958 100644 --- a/packages/ciphernode/keyshare/src/keyshare.rs +++ b/packages/ciphernode/keyshare/src/keyshare.rs @@ -9,6 +9,7 @@ use enclave_core::{ use fhe::{DecryptCiphertext, Fhe}; use serde::{Deserialize, Serialize}; use std::sync::Arc; +use tracing::warn; pub struct Keyshare { fhe: Arc, @@ -84,6 +85,7 @@ impl Handler for Keyshare { EnclaveEvent::CiphernodeSelected { data, .. } => ctx.notify(data), EnclaveEvent::CiphertextOutputPublished { data, .. } => ctx.notify(data), EnclaveEvent::E3RequestComplete { .. } => ctx.notify(Die), + EnclaveEvent::Shutdown { .. } => ctx.notify(Die), _ => (), } } @@ -162,6 +164,7 @@ impl Handler for Keyshare { impl Handler for Keyshare { type Result = (); fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { + warn!("Keyshare is shutting down"); ctx.stop() } } diff --git a/packages/ciphernode/router/src/ciphernode_selector.rs b/packages/ciphernode/router/src/ciphernode_selector.rs index 236ff0fe..b0dbd08c 100644 --- a/packages/ciphernode/router/src/ciphernode_selector.rs +++ b/packages/ciphernode/router/src/ciphernode_selector.rs @@ -1,7 +1,7 @@ /// CiphernodeSelector is an actor that determines if a ciphernode is part of a committee and if so /// forwards a CiphernodeSelected event to the event bus use actix::prelude::*; -use enclave_core::{CiphernodeSelected, EnclaveEvent, EventBus, Subscribe}; +use enclave_core::{CiphernodeSelected, E3Requested, EnclaveEvent, EventBus, Shutdown, Subscribe}; use sortition::{GetHasNode, Sortition}; use tracing::info; @@ -28,24 +28,32 @@ impl CiphernodeSelector { let addr = CiphernodeSelector::new(bus.clone(), sortition, address).start(); bus.do_send(Subscribe::new("E3Requested", addr.clone().recipient())); + bus.do_send(Subscribe::new("Shutdown", addr.clone().recipient())); addr } } impl Handler for CiphernodeSelector { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + match msg { + EnclaveEvent::E3Requested { data, .. } => ctx.notify(data), + EnclaveEvent::Shutdown { data, .. } => ctx.notify(data), + _ => (), + } + } +} + +impl Handler for CiphernodeSelector { type Result = ResponseFuture<()>; - fn handle(&mut self, event: EnclaveEvent, _ctx: &mut Self::Context) -> Self::Result { + fn handle(&mut self, data: E3Requested, _ctx: &mut Self::Context) -> Self::Result { let address = self.address.clone(); let sortition = self.sortition.clone(); let bus = self.bus.clone(); Box::pin(async move { - let EnclaveEvent::E3Requested { data, .. } = event else { - return; - }; - let seed = data.seed; let size = data.threshold_m; @@ -70,3 +78,11 @@ impl Handler for CiphernodeSelector { }) } } + +impl Handler for CiphernodeSelector { + type Result = (); + fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { + tracing::info!("Killing CiphernodeSelector"); + ctx.stop(); + } +} diff --git a/packages/ciphernode/router/src/context.rs b/packages/ciphernode/router/src/context.rs index 22d2e4d1..c3643132 100644 --- a/packages/ciphernode/router/src/context.rs +++ b/packages/ciphernode/router/src/context.rs @@ -82,6 +82,14 @@ impl E3RequestContext { }); } + pub fn forward_message_now(&self, msg: &EnclaveEvent) { + self.recipients().into_iter().for_each(|(_, recipient)| { + if let Some(act) = recipient { + act.do_send(msg.clone()); + } + }); + } + /// Accept a DataStore ID and a Keystore actor address pub fn set_keyshare(&mut self, value: Addr) { self.keyshare = Some(value); diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index daf57787..35d45c32 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -3,6 +3,7 @@ use crate::E3RequestContext; use crate::E3RequestContextParams; use crate::E3RequestContextSnapshot; use crate::RepositoriesFactory; +use actix::AsyncContext; use actix::{Actor, Addr, Context, Handler}; use anyhow::*; use async_trait::async_trait; @@ -12,6 +13,7 @@ use data::FromSnapshotWithParams; use data::Repository; use data::Snapshot; use enclave_core::E3RequestComplete; +use enclave_core::Shutdown; use enclave_core::{E3id, EnclaveEvent, EventBus, Subscribe}; use serde::Deserialize; use serde::Serialize; @@ -105,7 +107,13 @@ impl Actor for E3RequestRouter { impl Handler for E3RequestRouter { type Result = (); - fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EnclaveEvent, ctx: &mut Self::Context) -> Self::Result { + // If we are shuttomg down then bail on anything else + if let EnclaveEvent::Shutdown { data, .. } = msg { + ctx.notify(data); + return; + } + let Some(e3_id) = msg.get_e3_id() else { return; }; @@ -155,6 +163,16 @@ impl Handler for E3RequestRouter { } } +impl Handler for E3RequestRouter { + type Result = (); + fn handle(&mut self, msg: Shutdown, _ctx: &mut Self::Context) -> Self::Result { + let shutdown_evt = EnclaveEvent::from(msg); + for (_, ctx) in self.contexts.iter() { + ctx.forward_message_now(&shutdown_evt) + } + } +} + #[derive(Serialize, Deserialize)] pub struct E3RequestRouterSnapshot { contexts: Vec, diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 4c92d461..30e4c1bf 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -2,7 +2,7 @@ use data::{DataStore, InMemStore}; use enclave_core::{ CiphernodeAdded, CiphernodeSelected, CiphertextOutputPublished, DecryptionshareCreated, E3RequestComplete, E3Requested, E3id, EnclaveEvent, EventBus, GetHistory, KeyshareCreated, - OrderedSet, PlaintextAggregated, PublicKeyAggregated, ResetHistory, Seed, + OrderedSet, PlaintextAggregated, PublicKeyAggregated, ResetHistory, Seed, Shutdown, }; use fhe::{setup_crp_params, ParamsWithCrp, SharedRng}; use logger::SimpleLogger; @@ -14,7 +14,7 @@ use router::{ use sortition::Sortition; use actix::prelude::*; -use alloy::primitives::Address; +use alloy::{primitives::Address, signers::k256::sha2::digest::Reset}; use anyhow::*; use fhe_rs::{ bfv::{BfvParameters, Ciphertext, Encoding, Plaintext, PublicKey, SecretKey}, @@ -29,22 +29,31 @@ use tokio::sync::Mutex; use tokio::{sync::mpsc::channel, time::sleep}; // Simulating a local node +type LocalCiphernodeTuple = ( + String, // Address + Addr, + Addr, + Addr, + Addr, +); + async fn setup_local_ciphernode( bus: &Addr, rng: &SharedRng, logging: bool, addr: &str, -) -> Result<()> { + data: Option>, +) -> Result { // create data actor for saving data - let data_actor = InMemStore::new(logging).start(); // TODO: Use a sled backed Data Actor - let store = DataStore::from_in_mem(data_actor); + let data_actor = data.unwrap_or_else(|| InMemStore::new(logging).start()); + let store = DataStore::from_in_mem(&data_actor); let repositories = store.repositories(); // create ciphernode actor for managing ciphernode flow let sortition = Sortition::attach(bus.clone(), repositories.sortition()); CiphernodeSelector::attach(bus.clone(), sortition.clone(), addr); - E3RequestRouter::builder(bus.clone(), store) + let router = E3RequestRouter::builder(bus.clone(), store) .add_feature(FheFeature::create(rng.clone())) .add_feature(PublicKeyAggregatorFeature::create( bus.clone(), @@ -58,8 +67,8 @@ async fn setup_local_ciphernode( .build() .await?; - SimpleLogger::attach(addr, bus.clone()); - Ok(()) + let logger = SimpleLogger::attach(addr, bus.clone()); + Ok((addr.to_owned(), data_actor, sortition, router, logger)) } fn generate_pk_share( @@ -151,14 +160,15 @@ async fn create_local_ciphernodes( bus: &Addr, rng: &SharedRng, count: u32, -) -> Result> { +) -> Result> { let eth_addrs = create_random_eth_addrs(count); - + let mut result = vec![]; for addr in ð_addrs { - setup_local_ciphernode(&bus, &rng, true, addr).await?; + let tuple = setup_local_ciphernode(&bus, &rng, true, addr, None).await?; + result.push(tuple); } - Ok(eth_addrs) + Ok(result) } fn encrypt_ciphertext( @@ -246,9 +256,14 @@ fn to_decryptionshare_events( result } -#[actix::test] -async fn test_public_key_aggregation_and_decryption() -> Result<()> { - // Setup +fn get_common_setup() -> Result<( + Addr, + SharedRng, + Seed, + Arc, + CommonRandomPoly, + E3id, +)> { let bus = EventBus::new(true).start(); let rng = create_shared_rng_from_u64(42); let seed = create_seed_from_u64(123); @@ -256,9 +271,20 @@ async fn test_public_key_aggregation_and_decryption() -> Result<()> { let crpoly = CommonRandomPoly::deserialize(&crp_bytes.clone(), ¶ms)?; let e3_id = E3id::new("1234"); + Ok((bus, rng, seed, params, crpoly, e3_id)) +} + +#[actix::test] +async fn test_public_key_aggregation_and_decryption() -> Result<()> { + // Setup + let (bus, rng, seed, params, crpoly, e3_id) = get_common_setup()?; // Setup actual ciphernodes and dispatch add events - let eth_addrs = create_local_ciphernodes(&bus, &rng, 3).await?; + let ciphernode_addrs = create_local_ciphernodes(&bus, &rng, 3).await?; + let eth_addrs = ciphernode_addrs + .iter() + .map(|tup| tup.0.to_owned()) + .collect(); let add_events = add_ciphernodes(&bus, ð_addrs).await?; let e3_request_event = EnclaveEvent::from(E3Requested { e3_id: e3_id.clone(), @@ -355,7 +381,86 @@ async fn test_public_key_aggregation_and_decryption() -> Result<()> { } #[actix::test] -async fn test_actors_all_die_when_requested() {} +async fn test_stopped_keyshares_retain_state() -> Result<()> { + let (bus, rng, seed, params, crpoly, e3_id) = get_common_setup()?; + + let eth_addrs = create_random_eth_addrs(2); + + let cn1 = setup_local_ciphernode(&bus, &rng, true, ð_addrs[0], None).await?; + let cn2 = setup_local_ciphernode(&bus, &rng, true, ð_addrs[1], None).await?; + add_ciphernodes(&bus, ð_addrs).await?; + + sleep(Duration::from_millis(1)).await; + + // Send e3request + let e3_request_event = EnclaveEvent::from(E3Requested { + e3_id: e3_id.clone(), + threshold_m: 2, + seed: seed.clone(), + params: params.to_bytes(), + src_chain_id: 1, + }); + bus.send(e3_request_event.clone()).await?; + + sleep(Duration::from_millis(1)).await; + + let history = bus.send(GetHistory).await?; + + // SEND SHUTDOWN! + bus.send(EnclaveEvent::from(Shutdown)).await?; + + // Reset history + bus.send(ResetHistory).await?; + sleep(Duration::from_millis(1)).await; + + // Check event count is correct + assert_eq!(history.len(), 7); + + // Get the address and the data actor from the two ciphernodes + // and rehydrate them to new actors + let (addr1, data1, ..) = cn1; + let (addr2, data2, ..) = cn2; + + // Apply the address and data node to two new actors + // Here we test that hydration occurred sucessfully + setup_local_ciphernode(&bus, &rng, true, &addr1, Some(data1)).await?; + setup_local_ciphernode(&bus, &rng, true, &addr2, Some(data2)).await?; + // get the public key from history. + let pubkey: PublicKey = history + .iter() + .filter_map(|evt| match evt { + EnclaveEvent::KeyshareCreated { data, .. } => { + PublicKeyShare::deserialize(&data.pubkey, ¶ms, crpoly.clone()).ok() + } + _ => None, + }) + .aggregate()?; + + let raw_plaintext = vec![1234u64, 873827u64]; + let (ciphertext, expected) = encrypt_ciphertext(¶ms, pubkey, raw_plaintext)?; + + bus.send( + EnclaveEvent::from(CiphertextOutputPublished { + ciphertext_output: ciphertext.to_bytes(), + e3_id: e3_id.clone(), + }) + .clone(), + ) + .await?; + + sleep(Duration::from_millis(1)).await; + + let history = bus.send(GetHistory).await?; + + let actual = history.iter().find_map(|evt| match evt { + EnclaveEvent::PlaintextAggregated { data, .. } => Some(data.decrypted_output.clone()), + _ => None, + }); + + assert_eq!(actual, Some(expected)); + + Ok(()) +} #[actix::test] async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { From bd79b8386baaaff105405e1a740a8fa41c906350 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Mon, 21 Oct 2024 21:24:58 +1100 Subject: [PATCH 22/49] Formatting --- packages/ciphernode/core/src/events.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/ciphernode/core/src/events.rs b/packages/ciphernode/core/src/events.rs index 925028f3..c68183bf 100644 --- a/packages/ciphernode/core/src/events.rs +++ b/packages/ciphernode/core/src/events.rs @@ -212,7 +212,6 @@ pub trait FromError { fn from_error(err_type: EnclaveErrorType, error: Self::Error) -> Self; } - // TODO: These From traits should be handled by a macro impl From for EnclaveEvent { fn from(data: KeyshareCreated) -> Self { @@ -522,7 +521,6 @@ impl Display for Die { } } - /// Represents a shutdown event triggered by SIG TERM #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] From 596f8ed9cb133164a565a512231dd5c0c09b2156 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Mon, 21 Oct 2024 21:29:55 +1100 Subject: [PATCH 23/49] Remove sleeps --- .../tests/test_aggregation_and_decryption.rs | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 30e4c1bf..a3896f26 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -390,19 +390,18 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { let cn2 = setup_local_ciphernode(&bus, &rng, true, ð_addrs[1], None).await?; add_ciphernodes(&bus, ð_addrs).await?; - sleep(Duration::from_millis(1)).await; - // Send e3request - let e3_request_event = EnclaveEvent::from(E3Requested { - e3_id: e3_id.clone(), - threshold_m: 2, - seed: seed.clone(), - params: params.to_bytes(), - src_chain_id: 1, - }); - bus.send(e3_request_event.clone()).await?; - - sleep(Duration::from_millis(1)).await; + bus.send( + EnclaveEvent::from(E3Requested { + e3_id: e3_id.clone(), + threshold_m: 2, + seed: seed.clone(), + params: params.to_bytes(), + src_chain_id: 1, + }) + .clone(), + ) + .await?; let history = bus.send(GetHistory).await?; @@ -411,7 +410,6 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { // Reset history bus.send(ResetHistory).await?; - sleep(Duration::from_millis(1)).await; // Check event count is correct assert_eq!(history.len(), 7); @@ -436,9 +434,9 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { }) .aggregate()?; + // Publish the ciphertext let raw_plaintext = vec![1234u64, 873827u64]; let (ciphertext, expected) = encrypt_ciphertext(¶ms, pubkey, raw_plaintext)?; - bus.send( EnclaveEvent::from(CiphertextOutputPublished { ciphertext_output: ciphertext.to_bytes(), @@ -448,8 +446,6 @@ async fn test_stopped_keyshares_retain_state() -> Result<()> { ) .await?; - sleep(Duration::from_millis(1)).await; - let history = bus.send(GetHistory).await?; let actual = history.iter().find_map(|evt| match evt { From 9f0831fcd2d99f708afa0341f34adef97e154dcd Mon Sep 17 00:00:00 2001 From: ktdlr Date: Mon, 21 Oct 2024 22:42:15 +1100 Subject: [PATCH 24/49] Add option input to data location --- packages/ciphernode/Cargo.lock | 115 ++++++++++++++++-- packages/ciphernode/Cargo.toml | 1 + packages/ciphernode/core/src/events.rs | 1 + packages/ciphernode/data/Cargo.toml | 2 + packages/ciphernode/data/src/data_store.rs | 31 +++-- packages/ciphernode/data/src/lib.rs | 2 + packages/ciphernode/data/src/sled_store.rs | 60 +++++++++ .../ciphernode/enclave/src/bin/aggregator.rs | 3 + packages/ciphernode/enclave/src/main.rs | 5 +- .../ciphernode/enclave_node/src/aggregator.rs | 9 +- .../ciphernode/enclave_node/src/app_config.rs | 2 - .../ciphernode/enclave_node/src/ciphernode.rs | 14 ++- 12 files changed, 212 insertions(+), 33 deletions(-) create mode 100644 packages/ciphernode/data/src/sled_store.rs diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index efc250b2..257b0e7d 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -20,7 +20,7 @@ dependencies = [ "futures-util", "log", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "pin-project-lite", "smallvec", "tokio", @@ -1746,6 +1746,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -1755,6 +1764,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -1837,7 +1855,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.10", ] [[package]] @@ -1849,7 +1867,9 @@ dependencies = [ "anyhow", "async-trait", "bincode", + "enclave-core", "serde", + "sled", ] [[package]] @@ -2402,6 +2422,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "funty" version = "2.0.0" @@ -2570,6 +2600,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42012b0f064e01aa58b545fe3727f90f7dd4020f4a3ea735b50344965f5a57e9" +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -2754,7 +2793,7 @@ dependencies = [ "ipconfig", "lru-cache", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "rand", "resolv-conf", "smallvec", @@ -3301,7 +3340,7 @@ dependencies = [ "multihash", "multistream-select", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "pin-project", "quick-protobuf", "rand", @@ -3329,7 +3368,7 @@ dependencies = [ "multihash", "multistream-select", "once_cell", - "parking_lot", + "parking_lot 0.12.3", "pin-project", "quick-protobuf", "rand", @@ -3354,7 +3393,7 @@ dependencies = [ "hickory-resolver", "libp2p-core 0.41.3", "libp2p-identity", - "parking_lot", + "parking_lot 0.12.3", "smallvec", "tracing", ] @@ -3602,7 +3641,7 @@ dependencies = [ "libp2p-core 0.41.3", "libp2p-identity", "libp2p-tls", - "parking_lot", + "parking_lot 0.12.3", "quinn", "rand", "ring 0.17.8", @@ -4372,6 +4411,17 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -4379,7 +4429,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.10", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -4390,7 +4454,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.3", "smallvec", "windows-targets 0.52.6", ] @@ -4709,7 +4773,7 @@ checksum = "504ee9ff529add891127c4827eb481bd69dc0ebc72e9a682e187db4caa60c3ca" dependencies = [ "dtoa", "itoa", - "parking_lot", + "parking_lot 0.12.3", "prometheus-client-derive-encode", ] @@ -4967,6 +5031,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.3" @@ -5567,6 +5640,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "sled" +version = "0.34.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.2", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -5970,7 +6059,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", + "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", "socket2 0.5.7", @@ -6826,7 +6915,7 @@ dependencies = [ "futures", "log", "nohash-hasher", - "parking_lot", + "parking_lot 0.12.3", "pin-project", "rand", "static_assertions", @@ -6841,7 +6930,7 @@ dependencies = [ "futures", "log", "nohash-hasher", - "parking_lot", + "parking_lot 0.12.3", "pin-project", "rand", "static_assertions", diff --git a/packages/ciphernode/Cargo.toml b/packages/ciphernode/Cargo.toml index a2884404..8541e9a6 100644 --- a/packages/ciphernode/Cargo.toml +++ b/packages/ciphernode/Cargo.toml @@ -43,6 +43,7 @@ num = "0.4.3" rand_chacha = "0.3.1" rand = "0.8.5" serde = { version = "1.0.208", features = ["derive"] } +sled = "0.34.7" sha2 = "0.10.8" tokio = { version = "1.38", features = ["full"] } tracing = "0.1.37" diff --git a/packages/ciphernode/core/src/events.rs b/packages/ciphernode/core/src/events.rs index c68183bf..3fa72cd2 100644 --- a/packages/ciphernode/core/src/events.rs +++ b/packages/ciphernode/core/src/events.rs @@ -566,6 +566,7 @@ pub enum EnclaveErrorType { PlaintextAggregation, Decryption, Sortition, + Data } impl EnclaveError { diff --git a/packages/ciphernode/data/Cargo.toml b/packages/ciphernode/data/Cargo.toml index 387d1ffd..e1b13cc8 100644 --- a/packages/ciphernode/data/Cargo.toml +++ b/packages/ciphernode/data/Cargo.toml @@ -7,8 +7,10 @@ repository = "https://github.com/gnosisguild/enclave/packages/ciphernode" [dependencies] actix = { workspace = true } +enclave-core = { path = "../core" } anyhow = { workspace = true } serde = { workspace = true } +sled = { workspace = true } bincode = { workspace = true } async-trait = { workspace = true } diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index 4ba59060..c5d1918e 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -1,4 +1,4 @@ -use crate::{InMemStore, IntoKey}; +use crate::{InMemStore, IntoKey, SledStore}; use actix::{Addr, Message, Recipient}; use anyhow::Result; use serde::{Deserialize, Serialize}; @@ -63,15 +63,6 @@ impl DataStore { self.insert.do_send(msg) } - /// Construct a data store from an InMemStore actor - pub fn from_in_mem(addr: &Addr) -> Self { - Self { - get: addr.clone().recipient(), - insert: addr.clone().recipient(), - scope: vec![], - } - } - /// Get the scope as a string pub fn get_scope(&self) -> Result { Ok(String::from_utf8(self.scope.clone())?) @@ -118,3 +109,23 @@ impl DataStore { } } } + +impl From<&Addr> for DataStore { + fn from(addr: &Addr) -> Self { + Self { + get: addr.clone().recipient(), + insert: addr.clone().recipient(), + scope: vec![], + } + } +} + +impl From<&Addr> for DataStore { + fn from(addr: &Addr) -> Self { + Self { + get: addr.clone().recipient(), + insert: addr.clone().recipient(), + scope: vec![], + } + } +} diff --git a/packages/ciphernode/data/src/lib.rs b/packages/ciphernode/data/src/lib.rs index 38e2c077..1b6042a5 100644 --- a/packages/ciphernode/data/src/lib.rs +++ b/packages/ciphernode/data/src/lib.rs @@ -3,9 +3,11 @@ mod in_mem; mod into_key; mod repository; mod snapshot; +mod sled_store; pub use data_store::*; pub use in_mem::*; pub use into_key::IntoKey; pub use repository::*; pub use snapshot::*; +pub use sled_store::*; diff --git a/packages/ciphernode/data/src/sled_store.rs b/packages/ciphernode/data/src/sled_store.rs new file mode 100644 index 00000000..cfe1ead4 --- /dev/null +++ b/packages/ciphernode/data/src/sled_store.rs @@ -0,0 +1,60 @@ +use crate::{Get, Insert}; +use actix::{Actor, Addr, Handler}; +use anyhow::{Context, Result}; +use enclave_core::{BusError, EnclaveErrorType, EventBus}; +use sled::{Db, IVec}; + +pub struct SledStore { + db: Db, + bus: Addr, +} + +impl Actor for SledStore { + type Context = actix::Context; +} + +impl SledStore { + pub fn new(bus: &Addr, path: &str) -> Result { + let db = sled::open(path).context("could not open db")?; + Ok(Self { + db, + bus: bus.clone(), + }) + } +} + +impl Handler for SledStore { + type Result = (); + + fn handle(&mut self, event: Insert, _: &mut Self::Context) -> Self::Result { + match self + .db + .insert(event.key(), event.value()) + .context("Could not insert data into db") + { + Err(err) => self.bus.err(EnclaveErrorType::Data, err), + _ => (), + } + } +} + +impl Handler for SledStore { + type Result = Option>; + + fn handle(&mut self, event: Get, _: &mut Self::Context) -> Self::Result { + let key = event.key(); + let str_key = String::from_utf8_lossy(&key).into_owned(); + let res: Result> = self + .db + .get(key) + .context(format!("Failed to fetch {}", str_key)); + + return match res { + Ok(value) => value.map(|v| v.to_vec()), + Err(err) => { + self.bus.err(EnclaveErrorType::Data, err); + None + } + }; + } +} diff --git a/packages/ciphernode/enclave/src/bin/aggregator.rs b/packages/ciphernode/enclave/src/bin/aggregator.rs index 4d628b36..c6058be3 100644 --- a/packages/ciphernode/enclave/src/bin/aggregator.rs +++ b/packages/ciphernode/enclave/src/bin/aggregator.rs @@ -15,6 +15,8 @@ struct Args { pub pubkey_write_path: Option, #[arg(short, long = "plaintext-write-path")] pub plaintext_write_path: Option, + #[arg(short, long = "data-location")] + pub data_location: Option, } #[actix_rt::main] @@ -27,6 +29,7 @@ async fn main() -> Result<(), Box> { config, args.pubkey_write_path.as_deref(), args.plaintext_write_path.as_deref(), + args.data_location.as_deref(), ) .await?; let _ = tokio::join!(handle); diff --git a/packages/ciphernode/enclave/src/main.rs b/packages/ciphernode/enclave/src/main.rs index 39bb2105..e4ab7ee9 100644 --- a/packages/ciphernode/enclave/src/main.rs +++ b/packages/ciphernode/enclave/src/main.rs @@ -26,6 +26,8 @@ pub struct Args { pub address: String, #[arg(short, long)] pub config: String, + #[arg(short, long = "data-location")] + pub data_location: Option, } #[actix_rt::main] @@ -37,7 +39,8 @@ async fn main() -> Result<(), Box> { let address = Address::parse_checksummed(&args.address, None).expect("Invalid address"); info!("LAUNCHING CIPHERNODE: ({})", address); let config = load_config(&args.config)?; - let (_, handle) = MainCiphernode::attach(config, address).await?; + let (_, handle) = + MainCiphernode::attach(config, address, args.data_location.as_deref()).await?; let _ = tokio::join!(handle); Ok(()) } diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 328ed40d..6a2bc31f 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -1,6 +1,6 @@ use actix::{Actor, Addr, Context}; use anyhow::Result; -use data::{DataStore, InMemStore}; +use data::{DataStore, InMemStore, SledStore}; use enclave_core::EventBus; use evm::{ helpers::pull_eth_signer_from_env, CiphernodeRegistrySol, EnclaveSol, RegistryFilterSol, @@ -49,13 +49,18 @@ impl MainAggregator { config: AppConfig, pubkey_write_path: Option<&str>, plaintext_write_path: Option<&str>, + data_location: Option<&str>, ) -> Result<(Addr, JoinHandle<()>)> { let bus = EventBus::new(true).start(); let rng = Arc::new(Mutex::new( rand_chacha::ChaCha20Rng::from_rng(OsRng).expect("Failed to create RNG"), )); - let store = DataStore::from_in_mem(&InMemStore::new(true).start()); + let store: DataStore = match data_location { + Some(loc) => (&SledStore::new(&bus, loc)?.start()).into(), + None => (&InMemStore::new(true).start()).into(), + }; + let repositories = store.repositories(); let sortition = Sortition::attach(bus.clone(), repositories.sortition()); let signer = pull_eth_signer_from_env("PRIVATE_KEY").await?; diff --git a/packages/ciphernode/enclave_node/src/app_config.rs b/packages/ciphernode/enclave_node/src/app_config.rs index 42cd8587..8d1bdef2 100644 --- a/packages/ciphernode/enclave_node/src/app_config.rs +++ b/packages/ciphernode/enclave_node/src/app_config.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use serde::Deserialize; #[derive(Debug, Deserialize)] diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 2f74f37b..47b459ee 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -1,7 +1,7 @@ use actix::{Actor, Addr, Context}; use alloy::primitives::Address; use anyhow::Result; -use data::{DataStore, InMemStore}; +use data::{DataStore, InMemStore, SledStore}; use enclave_core::EventBus; use evm::{CiphernodeRegistrySol, EnclaveSolReader}; use logger::SimpleLogger; @@ -9,8 +9,7 @@ use p2p::P2p; use rand::SeedableRng; use rand_chacha::rand_core::OsRng; use router::{ - CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, Repositories, - RepositoriesFactory, + CiphernodeSelector, E3RequestRouter, FheFeature, KeyshareFeature, RepositoriesFactory, }; use sortition::Sortition; use std::sync::{Arc, Mutex}; @@ -55,13 +54,18 @@ impl MainCiphernode { pub async fn attach( config: AppConfig, address: Address, + data_location: Option<&str>, ) -> Result<(Addr, JoinHandle<()>)> { let rng = Arc::new(Mutex::new( rand_chacha::ChaCha20Rng::from_rng(OsRng).expect("Failed to create RNG"), )); let bus = EventBus::new(true).start(); - // TODO: switch to Sled actor - let store = DataStore::from_in_mem(&InMemStore::new(true).start()); + + let store: DataStore = match data_location { + Some(loc) => (&SledStore::new(&bus, loc)?.start()).into(), + None => (&InMemStore::new(true).start()).into(), + }; + let repositories = store.repositories(); let sortition = Sortition::attach(bus.clone(), repositories.sortition()); From 972c767c49e5d2e9612619085e4a613e84cd57d8 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 07:42:36 +1100 Subject: [PATCH 25/49] Add data-location to integration test --- tests/basic_integration/output/.gitignore | 1 + tests/basic_integration/test.sh | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/basic_integration/output/.gitignore b/tests/basic_integration/output/.gitignore index 730ffd4a..93610434 100644 --- a/tests/basic_integration/output/.gitignore +++ b/tests/basic_integration/output/.gitignore @@ -1,3 +1,4 @@ *.bin plaintext.txt *.hex +*.db diff --git a/tests/basic_integration/test.sh b/tests/basic_integration/test.sh index 70be5115..3c1db2a4 100755 --- a/tests/basic_integration/test.sh +++ b/tests/basic_integration/test.sh @@ -76,6 +76,12 @@ waiton-files() { done } +launch_ciphernode() { + local address="$1" + heading "Launch ciphernode $address" + yarn ciphernode:launch --address "$address" --config "$SCRIPT_DIR/lib/ciphernode_config.yaml" --data-location "$SCRIPT_DIR/output/$address.db" & +} + pkill -9 -f "target/debug/enclave" || true pkill -9 -f "hardhat node" || true pkill -9 -f "target/debug/aggregator" || true @@ -98,17 +104,10 @@ done # Launch 4 ciphernodes -heading "Launch ciphernode $CIPHERNODE_ADDRESS_1" -yarn ciphernode:launch --address $CIPHERNODE_ADDRESS_1 --config "$SCRIPT_DIR/lib/ciphernode_config.yaml" & - -heading "Launch ciphernode $CIPHERNODE_ADDRESS_2" -yarn ciphernode:launch --address $CIPHERNODE_ADDRESS_2 --config "$SCRIPT_DIR/lib/ciphernode_config.yaml" & - -heading "Launch ciphernode $CIPHERNODE_ADDRESS_3" -yarn ciphernode:launch --address $CIPHERNODE_ADDRESS_3 --config "$SCRIPT_DIR/lib/ciphernode_config.yaml" & - -heading "Launch ciphernode $CIPHERNODE_ADDRESS_4" -yarn ciphernode:launch --address $CIPHERNODE_ADDRESS_4 --config "$SCRIPT_DIR/lib/ciphernode_config.yaml" & +launch_ciphernode "$CIPHERNODE_ADDRESS_1" +launch_ciphernode "$CIPHERNODE_ADDRESS_2" +launch_ciphernode "$CIPHERNODE_ADDRESS_3" +launch_ciphernode "$CIPHERNODE_ADDRESS_4" # NOTE: This node is configured to be an aggregator PRIVATE_KEY=$PRIVATE_KEY yarn ciphernode:aggregator --config "$SCRIPT_DIR/lib/ciphernode_config.yaml" --pubkey-write-path "$SCRIPT_DIR/output/pubkey.bin" --plaintext-write-path "$SCRIPT_DIR/output/plaintext.txt" & From 562fdf0d111219fedfdae0bfba8764de7e2ec64a Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 09:14:20 +1100 Subject: [PATCH 26/49] Extract event reader and add shutdown channel --- packages/ciphernode/core/src/events.rs | 2 +- packages/ciphernode/data/src/lib.rs | 4 +- .../evm/src/ciphernode_registry_sol.rs | 58 ++------------ .../ciphernode/evm/src/enclave_sol_reader.rs | 56 ++----------- packages/ciphernode/evm/src/event_reader.rs | 78 +++++++++++++++++++ packages/ciphernode/evm/src/helpers.rs | 33 +++++--- packages/ciphernode/evm/src/lib.rs | 2 + tests/basic_integration/test.sh | 17 +++- 8 files changed, 136 insertions(+), 114 deletions(-) create mode 100644 packages/ciphernode/evm/src/event_reader.rs diff --git a/packages/ciphernode/core/src/events.rs b/packages/ciphernode/core/src/events.rs index 3fa72cd2..b032bdde 100644 --- a/packages/ciphernode/core/src/events.rs +++ b/packages/ciphernode/core/src/events.rs @@ -566,7 +566,7 @@ pub enum EnclaveErrorType { PlaintextAggregation, Decryption, Sortition, - Data + Data, } impl EnclaveError { diff --git a/packages/ciphernode/data/src/lib.rs b/packages/ciphernode/data/src/lib.rs index 1b6042a5..b3f1ea93 100644 --- a/packages/ciphernode/data/src/lib.rs +++ b/packages/ciphernode/data/src/lib.rs @@ -2,12 +2,12 @@ mod data_store; mod in_mem; mod into_key; mod repository; -mod snapshot; mod sled_store; +mod snapshot; pub use data_store::*; pub use in_mem::*; pub use into_key::IntoKey; pub use repository::*; -pub use snapshot::*; pub use sled_store::*; +pub use snapshot::*; diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index 033c41c6..f4f4b7b3 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -1,16 +1,13 @@ -use actix::{Actor, Addr, AsyncContext, Recipient, WrapFuture}; +use crate::EvmEventReader; +use actix::Addr; use alloy::{ - eips::BlockNumberOrTag, - primitives::{Address, LogData, B256}, - rpc::types::Filter, + primitives::{LogData, B256}, sol, sol_types::SolEvent, }; use anyhow::Result; use enclave_core::{EnclaveEvent, EventBus}; -use tracing::{error, info, trace}; - -use crate::helpers::{self, create_readonly_provider, ReadonlyProvider}; +use tracing::{error, trace}; sol!( #[sol(rpc)] @@ -65,7 +62,7 @@ impl From for EnclaveEvent { } } -fn extractor(data: &LogData, topic: Option<&B256>, _: u64) -> Option { +pub fn extractor(data: &LogData, topic: Option<&B256>, _: u64) -> Option { match topic { Some(&ICiphernodeRegistry::CiphernodeAdded::SIGNATURE_HASH) => { let Ok(event) = ICiphernodeRegistry::CiphernodeAdded::decode_log_data(data, true) @@ -95,58 +92,19 @@ fn extractor(data: &LogData, topic: Option<&B256>, _: u64) -> Option, -} +pub struct CiphernodeRegistrySolReader; impl CiphernodeRegistrySolReader { - pub async fn new( - bus: Addr, - contract_address: Address, - rpc_url: &str, - ) -> Result { - let provider = create_readonly_provider(rpc_url).await?; - - Ok(Self { - contract_address, - provider, - bus: bus.into(), - }) - } - pub async fn attach( bus: Addr, rpc_url: &str, contract_address: &str, - ) -> Result> { - let addr = - CiphernodeRegistrySolReader::new(bus.clone(), contract_address.parse()?, rpc_url) - .await? - .start(); - - info!(address=%contract_address, "CiphernodeRegistrySol is listening to address"); + ) -> Result> { + let addr = EvmEventReader::attach(bus, rpc_url, extractor, contract_address).await?; Ok(addr) } } -impl Actor for CiphernodeRegistrySolReader { - type Context = actix::Context; - fn started(&mut self, ctx: &mut Self::Context) { - let bus = self.bus.clone(); - let provider = self.provider.clone(); - let filter = Filter::new() - .address(self.contract_address) - .from_block(BlockNumberOrTag::Latest); - - ctx.spawn( - async move { helpers::stream_from_evm(provider, filter, bus, extractor).await } - .into_actor(self), - ); - } -} - pub struct CiphernodeRegistrySol; impl CiphernodeRegistrySol { pub async fn attach(bus: Addr, rpc_url: &str, contract_address: &str) -> Result<()> { diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index 7b29f6e3..e46c618c 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -1,13 +1,10 @@ -use crate::helpers::{self, create_readonly_provider, ReadonlyProvider}; -use actix::prelude::*; -use actix::{Addr, Recipient}; +use crate::EvmEventReader; +use actix::Addr; use alloy::primitives::{LogData, B256}; -use alloy::{ - eips::BlockNumberOrTag, primitives::Address, rpc::types::Filter, sol, sol_types::SolEvent, -}; +use alloy::{sol, sol_types::SolEvent}; use anyhow::Result; use enclave_core::{EnclaveEvent, EventBus}; -use tracing::{error, info, trace}; +use tracing::{error, trace}; sol!( #[sol(rpc)] @@ -52,7 +49,7 @@ impl From for EnclaveEvent { } } -fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option { +pub fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option { match topic { Some(&IEnclave::E3Requested::SIGNATURE_HASH) => { let Ok(event) = IEnclave::E3Requested::decode_log_data(data, true) else { @@ -80,52 +77,15 @@ fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option, -} +pub struct EnclaveSolReader; impl EnclaveSolReader { - pub async fn new( - bus: Addr, - contract_address: Address, - rpc_url: &str, - ) -> Result { - let provider = create_readonly_provider(rpc_url).await?; - Ok(Self { - contract_address, - provider, - bus: bus.into(), - }) - } - pub async fn attach( bus: Addr, rpc_url: &str, contract_address: &str, - ) -> Result> { - let addr = EnclaveSolReader::new(bus.clone(), contract_address.parse()?, rpc_url) - .await? - .start(); - - info!(address=%contract_address, "Evm is listening to address"); + ) -> Result> { + let addr = EvmEventReader::attach(bus, rpc_url, extractor, contract_address).await?; Ok(addr) } } - -impl Actor for EnclaveSolReader { - type Context = actix::Context; - fn started(&mut self, ctx: &mut Self::Context) { - let bus = self.bus.clone(); - let provider = self.provider.clone(); - let filter = Filter::new() - .address(self.contract_address) - .from_block(BlockNumberOrTag::Latest); - - ctx.spawn( - async move { helpers::stream_from_evm(provider, filter, bus, extractor).await } - .into_actor(self), - ); - } -} diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs new file mode 100644 index 00000000..679c9634 --- /dev/null +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -0,0 +1,78 @@ +use crate::helpers::{self, create_readonly_provider, ReadonlyProvider}; +use actix::prelude::*; +use actix::{Addr, Recipient}; +use alloy::primitives::{LogData, B256}; +use alloy::{ + eips::BlockNumberOrTag, primitives::Address, rpc::types::Filter, sol, sol_types::SolEvent, +}; +use anyhow::{anyhow, Result}; +use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus}; +use tokio::sync::oneshot; +use tracing::info; + +pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; + +/// Connects to Enclave.sol converting EVM events to EnclaveEvents +pub struct EvmEventReader { + provider: ReadonlyProvider, + contract_address: Address, + bus: Recipient, + extractor: fn(&LogData, Option<&B256>, u64) -> Option, + shutdown_rx: Option>, + shutdown_tx: oneshot::Sender<()>, +} + +impl EvmEventReader { + pub async fn new( + bus: Addr, + rpc_url: &str, + extractor: ExtractorFn, + contract_address: Address, + ) -> Result { + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let provider = create_readonly_provider(rpc_url).await?; + Ok(Self { + contract_address, + provider, + extractor, + bus: bus.into(), + shutdown_rx: Some(shutdown_rx), + shutdown_tx, + }) + } + + pub async fn attach( + bus: Addr, + rpc_url: &str, + extractor: ExtractorFn, + contract_address: &str, + ) -> Result> { + let addr = EvmEventReader::new(bus.clone(), rpc_url, extractor, contract_address.parse()?) + .await? + .start(); + + info!(address=%contract_address, "Evm is listening to address"); + Ok(addr) + } +} + +impl Actor for EvmEventReader { + type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + let bus = self.bus.clone(); + let provider = self.provider.clone(); + let filter = Filter::new() + .address(self.contract_address) + .from_block(BlockNumberOrTag::Latest); + let extractor = self.extractor; + let Some(shutdown) = self.shutdown_rx.take() else { + bus.err(EnclaveErrorType::Evm, anyhow!("shutdown already called")); + return; + }; + + ctx.spawn( + async move { helpers::stream_from_evm(provider, filter, bus, extractor, shutdown).await } + .into_actor(self), + ); + } +} diff --git a/packages/ciphernode/evm/src/helpers.rs b/packages/ciphernode/evm/src/helpers.rs index 680e0e39..a727347f 100644 --- a/packages/ciphernode/evm/src/helpers.rs +++ b/packages/ciphernode/evm/src/helpers.rs @@ -15,6 +15,7 @@ use alloy::{ use anyhow::{Context, Result}; use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent}; use futures_util::stream::StreamExt; +use tokio::{select, sync::oneshot}; use tracing::{info, trace}; pub async fn stream_from_evm( @@ -22,6 +23,7 @@ pub async fn stream_from_evm( filter: Filter, bus: Recipient, extractor: fn(&LogData, Option<&B256>, u64) -> Option, + mut shutdown: oneshot::Receiver<()>, ) { match provider .get_provider() @@ -31,15 +33,28 @@ pub async fn stream_from_evm( { Ok(subscription) => { let mut stream = subscription.into_stream(); - while let Some(log) = stream.next().await { - trace!("Received log from EVM"); - let Some(event) = extractor(log.data(), log.topic0(), provider.get_chain_id()) - else { - trace!("Failed to extract log from EVM"); - continue; - }; - info!("Extracted log from evm sending now."); - bus.do_send(event); + loop { + select! { + maybe_log = stream.next() => { + match maybe_log { + Some(log) => { + trace!("Received log from EVM"); + let Some(event) = extractor(log.data(), log.topic0(), provider.get_chain_id()) + else { + trace!("Failed to extract log from EVM"); + continue; + }; + info!("Extracted log from evm sending now."); + bus.do_send(event); + } + None => break, // Stream ended + } + } + _ = &mut shutdown => { + info!("Received shutdown signal, stopping EVM stream"); + break; + } + } } } Err(e) => { diff --git a/packages/ciphernode/evm/src/lib.rs b/packages/ciphernode/evm/src/lib.rs index 65a5f5bd..51d70288 100644 --- a/packages/ciphernode/evm/src/lib.rs +++ b/packages/ciphernode/evm/src/lib.rs @@ -4,9 +4,11 @@ mod enclave_sol_reader; mod enclave_sol_writer; pub mod helpers; mod registry_filter_sol; +mod event_reader; pub use ciphernode_registry_sol::{CiphernodeRegistrySol, CiphernodeRegistrySolReader}; pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; pub use registry_filter_sol::{RegistryFilterSol, RegistryFilterSolWriter}; +pub use event_reader::EvmEventReader; diff --git a/tests/basic_integration/test.sh b/tests/basic_integration/test.sh index 3c1db2a4..6fb671e5 100755 --- a/tests/basic_integration/test.sh +++ b/tests/basic_integration/test.sh @@ -79,7 +79,18 @@ waiton-files() { launch_ciphernode() { local address="$1" heading "Launch ciphernode $address" - yarn ciphernode:launch --address "$address" --config "$SCRIPT_DIR/lib/ciphernode_config.yaml" --data-location "$SCRIPT_DIR/output/$address.db" & + yarn ciphernode:launch \ + --address "$address" \ + --config "$SCRIPT_DIR/lib/ciphernode_config.yaml" \ + --data-location "$SCRIPT_DIR/output/$address.db" & +} + +launch_aggregator() { + local private_key="$1" + PRIVATE_KEY=$private_key yarn ciphernode:aggregator \ + --config "$SCRIPT_DIR/lib/ciphernode_config.yaml" \ + --pubkey-write-path "$SCRIPT_DIR/output/pubkey.bin" \ + --plaintext-write-path "$SCRIPT_DIR/output/plaintext.txt" & } pkill -9 -f "target/debug/enclave" || true @@ -108,9 +119,7 @@ launch_ciphernode "$CIPHERNODE_ADDRESS_1" launch_ciphernode "$CIPHERNODE_ADDRESS_2" launch_ciphernode "$CIPHERNODE_ADDRESS_3" launch_ciphernode "$CIPHERNODE_ADDRESS_4" - -# NOTE: This node is configured to be an aggregator -PRIVATE_KEY=$PRIVATE_KEY yarn ciphernode:aggregator --config "$SCRIPT_DIR/lib/ciphernode_config.yaml" --pubkey-write-path "$SCRIPT_DIR/output/pubkey.bin" --plaintext-write-path "$SCRIPT_DIR/output/plaintext.txt" & +launch_aggregator "$PRIVATE_KEY" sleep 1 From 5bee918c93c5b2d4a535cc74ac463d512fa35f7c Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 10:04:01 +1100 Subject: [PATCH 27/49] Tidy up shutdown --- packages/ciphernode/Cargo.lock | 1 + packages/ciphernode/enclave/Cargo.toml | 1 + .../ciphernode/enclave/src/bin/aggregator.rs | 8 ++++-- packages/ciphernode/enclave/src/main.rs | 8 ++++-- packages/ciphernode/enclave_node/src/lib.rs | 3 +- .../ciphernode/enclave_node/src/shutdown.rs | 25 +++++++++++++++++ packages/ciphernode/evm/src/event_reader.rs | 28 +++++++++++++++++-- 7 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 packages/ciphernode/enclave_node/src/shutdown.rs diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 257b0e7d..40c35dce 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -2109,6 +2109,7 @@ dependencies = [ name = "enclave" version = "0.1.0" dependencies = [ + "actix", "actix-rt", "alloy", "clap", diff --git a/packages/ciphernode/enclave/Cargo.toml b/packages/ciphernode/enclave/Cargo.toml index 064b2e7e..0da57db0 100644 --- a/packages/ciphernode/enclave/Cargo.toml +++ b/packages/ciphernode/enclave/Cargo.toml @@ -11,6 +11,7 @@ repository = "https://github.com/gnosisguild/enclave/packages/ciphernode" enclave_node = { path = "../enclave_node" } alloy = { workspace = true } clap = { workspace = true } +actix = { workspace = true } actix-rt = { workspace = true } tokio = { workspace = true } config = "0.14.0" diff --git a/packages/ciphernode/enclave/src/bin/aggregator.rs b/packages/ciphernode/enclave/src/bin/aggregator.rs index c6058be3..c54a5b94 100644 --- a/packages/ciphernode/enclave/src/bin/aggregator.rs +++ b/packages/ciphernode/enclave/src/bin/aggregator.rs @@ -1,6 +1,6 @@ use clap::Parser; use enclave::load_config; -use enclave_node::MainAggregator; +use enclave_node::{listen_for_shutdown, MainAggregator}; use tracing::info; #[derive(Parser, Debug)] @@ -32,6 +32,10 @@ async fn main() -> Result<(), Box> { args.data_location.as_deref(), ) .await?; - let _ = tokio::join!(handle); + + tokio::spawn(listen_for_shutdown(handle)); + + std::future::pending::<()>().await; + Ok(()) } diff --git a/packages/ciphernode/enclave/src/main.rs b/packages/ciphernode/enclave/src/main.rs index e4ab7ee9..62ee97e5 100644 --- a/packages/ciphernode/enclave/src/main.rs +++ b/packages/ciphernode/enclave/src/main.rs @@ -1,7 +1,7 @@ use alloy::primitives::Address; use clap::Parser; use enclave::load_config; -use enclave_node::MainCiphernode; +use enclave_node::{listen_for_shutdown, MainCiphernode}; use tracing::info; const OWO: &str = r#" @@ -41,6 +41,10 @@ async fn main() -> Result<(), Box> { let config = load_config(&args.config)?; let (_, handle) = MainCiphernode::attach(config, address, args.data_location.as_deref()).await?; - let _ = tokio::join!(handle); + + tokio::spawn(listen_for_shutdown(handle)); + + std::future::pending::<()>().await; + Ok(()) } diff --git a/packages/ciphernode/enclave_node/src/lib.rs b/packages/ciphernode/enclave_node/src/lib.rs index 96fffacb..7f475274 100644 --- a/packages/ciphernode/enclave_node/src/lib.rs +++ b/packages/ciphernode/enclave_node/src/lib.rs @@ -1,8 +1,9 @@ mod aggregator; mod app_config; mod ciphernode; +mod shutdown; pub use aggregator::*; pub use ciphernode::*; - pub use app_config::*; +pub use shutdown::*; diff --git a/packages/ciphernode/enclave_node/src/shutdown.rs b/packages/ciphernode/enclave_node/src/shutdown.rs new file mode 100644 index 00000000..34ee268f --- /dev/null +++ b/packages/ciphernode/enclave_node/src/shutdown.rs @@ -0,0 +1,25 @@ +use std::time::Duration; + +use actix::System; +use tokio::{signal::unix::{signal, SignalKind}, task::JoinHandle}; + +pub async fn listen_for_shutdown(handle: JoinHandle<()>) { + let mut sigterm = signal(SignalKind::terminate()) + .expect("Failed to create SIGTERM signal stream"); + + sigterm.recv().await; + println!("SIGTERM received, initiating graceful shutdown..."); + + System::current().stop(); + tokio::time::sleep(Duration::from_secs(5)).await; + + // Abort the spawned task + handle.abort(); + + // Wait for the task to finish + if let Err(e) = handle.await { + println!("Task error during shutdown: {:?}", e); + } + + println!("Graceful shutdown complete"); +} diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index 679c9634..7ce67710 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -6,7 +6,7 @@ use alloy::{ eips::BlockNumberOrTag, primitives::Address, rpc::types::Filter, sol, sol_types::SolEvent, }; use anyhow::{anyhow, Result}; -use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus}; +use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus, Shutdown, Subscribe}; use tokio::sync::oneshot; use tracing::info; @@ -19,7 +19,7 @@ pub struct EvmEventReader { bus: Recipient, extractor: fn(&LogData, Option<&B256>, u64) -> Option, shutdown_rx: Option>, - shutdown_tx: oneshot::Sender<()>, + shutdown_tx: Option>, } impl EvmEventReader { @@ -37,7 +37,7 @@ impl EvmEventReader { extractor, bus: bus.into(), shutdown_rx: Some(shutdown_rx), - shutdown_tx, + shutdown_tx: Some(shutdown_tx), }) } @@ -51,6 +51,9 @@ impl EvmEventReader { .await? .start(); + bus.send(Subscribe::new("Shutdown", addr.clone().into())) + .await?; + info!(address=%contract_address, "Evm is listening to address"); Ok(addr) } @@ -75,4 +78,23 @@ impl Actor for EvmEventReader { .into_actor(self), ); } + + fn stopping(&mut self, _: &mut Self::Context) -> Running { + if let Some(shutdown) = self.shutdown_tx.take() { + let _ = shutdown.send(()); + } + + Running::Stop + } +} + +impl Handler for EvmEventReader { + type Result = (); + fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { + if let EnclaveEvent::Shutdown { .. } = msg { + if let Some(shutdown) = self.shutdown_tx.take() { + let _ = shutdown.send(()); + } + } + } } From 2267c40be93c87e51fc2b1db75fe9efa479d42bd Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 11:28:09 +1100 Subject: [PATCH 28/49] Tidy up shutdown --- packages/ciphernode/Cargo.lock | 1 + packages/ciphernode/data/src/data_store.rs | 2 +- .../ciphernode/enclave/src/bin/aggregator.rs | 4 +- packages/ciphernode/enclave/src/main.rs | 4 +- packages/ciphernode/enclave_node/Cargo.toml | 1 + .../ciphernode/enclave_node/src/aggregator.rs | 10 ++--- .../ciphernode/enclave_node/src/ciphernode.rs | 8 ++-- .../ciphernode/enclave_node/src/shutdown.rs | 30 +++++++------ .../ciphernode/evm/src/enclave_sol_writer.rs | 3 +- packages/ciphernode/evm/src/event_reader.rs | 21 ++++------ packages/ciphernode/evm/src/helpers.rs | 42 ++++++++++++++++++- .../ciphernode/evm/src/registry_filter_sol.rs | 4 +- .../tests/test_aggregation_and_decryption.rs | 2 +- tests/basic_integration/test.sh | 5 +++ 14 files changed, 92 insertions(+), 45 deletions(-) diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 40c35dce..1d6499c5 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -2163,6 +2163,7 @@ dependencies = [ "sortition", "test-helpers", "tokio", + "tracing", ] [[package]] diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index c5d1918e..7028b090 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -79,7 +79,7 @@ impl DataStore { /// #[actix_rt::main] /// async fn main() -> Result<()>{ /// let addr = InMemStore::new(false).start(); - /// let store = DataStore::from_in_mem(&addr); + /// let store = DataStore::from(&addr); /// assert_eq!(store.base("//foo") /// .scope("bar") /// .scope("/baz") diff --git a/packages/ciphernode/enclave/src/bin/aggregator.rs b/packages/ciphernode/enclave/src/bin/aggregator.rs index c54a5b94..37eac82a 100644 --- a/packages/ciphernode/enclave/src/bin/aggregator.rs +++ b/packages/ciphernode/enclave/src/bin/aggregator.rs @@ -25,7 +25,7 @@ async fn main() -> Result<(), Box> { let args = Args::parse(); info!("LAUNCHING AGGREGATOR"); let config = load_config(&args.config)?; - let (_, handle) = MainAggregator::attach( + let (bus, handle) = MainAggregator::attach( config, args.pubkey_write_path.as_deref(), args.plaintext_write_path.as_deref(), @@ -33,7 +33,7 @@ async fn main() -> Result<(), Box> { ) .await?; - tokio::spawn(listen_for_shutdown(handle)); + tokio::spawn(listen_for_shutdown(bus.into(), handle)); std::future::pending::<()>().await; diff --git a/packages/ciphernode/enclave/src/main.rs b/packages/ciphernode/enclave/src/main.rs index 62ee97e5..488a6a72 100644 --- a/packages/ciphernode/enclave/src/main.rs +++ b/packages/ciphernode/enclave/src/main.rs @@ -39,10 +39,10 @@ async fn main() -> Result<(), Box> { let address = Address::parse_checksummed(&args.address, None).expect("Invalid address"); info!("LAUNCHING CIPHERNODE: ({})", address); let config = load_config(&args.config)?; - let (_, handle) = + let (bus, handle) = MainCiphernode::attach(config, address, args.data_location.as_deref()).await?; - tokio::spawn(listen_for_shutdown(handle)); + tokio::spawn(listen_for_shutdown(bus.into(), handle)); std::future::pending::<()>().await; diff --git a/packages/ciphernode/enclave_node/Cargo.toml b/packages/ciphernode/enclave_node/Cargo.toml index ea0e68bc..44ba409a 100644 --- a/packages/ciphernode/enclave_node/Cargo.toml +++ b/packages/ciphernode/enclave_node/Cargo.toml @@ -31,3 +31,4 @@ serde = { workspace = true } sortition = { path = "../sortition" } test-helpers = { path = "../test_helpers" } tokio = { workspace = true } +tracing = { workspace = true } diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 6a2bc31f..938c409f 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -32,14 +32,14 @@ pub struct MainAggregator { impl MainAggregator { pub fn new( - bus: Addr, + bus: &Addr, sortition: Addr, p2p: Addr, e3_manager: Addr, ) -> Self { Self { e3_manager, - bus, + bus: bus.clone(), sortition, p2p, } @@ -50,7 +50,7 @@ impl MainAggregator { pubkey_write_path: Option<&str>, plaintext_write_path: Option<&str>, data_location: Option<&str>, - ) -> Result<(Addr, JoinHandle<()>)> { + ) -> Result<(Addr, JoinHandle<()>)> { let bus = EventBus::new(true).start(); let rng = Arc::new(Mutex::new( rand_chacha::ChaCha20Rng::from_rng(OsRng).expect("Failed to create RNG"), @@ -118,8 +118,8 @@ impl MainAggregator { SimpleLogger::attach("AGG", bus.clone()); - let main_addr = MainAggregator::new(bus, sortition, p2p_addr, e3_manager).start(); - Ok((main_addr, join_handle)) + MainAggregator::new(&bus, sortition, p2p_addr, e3_manager).start(); + Ok((bus, join_handle)) } } diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 47b459ee..ba329dc2 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -55,7 +55,7 @@ impl MainCiphernode { config: AppConfig, address: Address, data_location: Option<&str>, - ) -> Result<(Addr, JoinHandle<()>)> { + ) -> Result<(Addr, JoinHandle<()>)> { let rng = Arc::new(Mutex::new( rand_chacha::ChaCha20Rng::from_rng(OsRng).expect("Failed to create RNG"), )); @@ -99,11 +99,11 @@ impl MainCiphernode { let nm = format!("CIPHER({})", &address.to_string()[0..5]); SimpleLogger::attach(&nm, bus.clone()); - let main_addr = MainCiphernode::new( - address, bus, store, sortition, selector, p2p_addr, e3_manager, + MainCiphernode::new( + address, bus.clone(), store, sortition, selector, p2p_addr, e3_manager, ) .start(); - Ok((main_addr, join_handle)) + Ok((bus, join_handle)) } } diff --git a/packages/ciphernode/enclave_node/src/shutdown.rs b/packages/ciphernode/enclave_node/src/shutdown.rs index 34ee268f..ce36c4ee 100644 --- a/packages/ciphernode/enclave_node/src/shutdown.rs +++ b/packages/ciphernode/enclave_node/src/shutdown.rs @@ -1,25 +1,29 @@ +use actix::Recipient; +use enclave_core::{EnclaveEvent, Shutdown}; use std::time::Duration; +use tokio::{ + signal::unix::{signal, SignalKind}, + task::JoinHandle, +}; +use tracing::{error, info}; -use actix::System; -use tokio::{signal::unix::{signal, SignalKind}, task::JoinHandle}; - -pub async fn listen_for_shutdown(handle: JoinHandle<()>) { - let mut sigterm = signal(SignalKind::terminate()) - .expect("Failed to create SIGTERM signal stream"); +pub async fn listen_for_shutdown(bus: Recipient, handle: JoinHandle<()>) { + let mut sigterm = + signal(SignalKind::terminate()).expect("Failed to create SIGTERM signal stream"); sigterm.recv().await; - println!("SIGTERM received, initiating graceful shutdown..."); + info!("SIGTERM received, initiating graceful shutdown..."); - System::current().stop(); - tokio::time::sleep(Duration::from_secs(5)).await; + // Stop the actor system + let _ = bus.send(EnclaveEvent::from(Shutdown)).await; // Abort the spawned task handle.abort(); + tokio::time::sleep(Duration::from_secs(2)).await; + // Wait for the task to finish - if let Err(e) = handle.await { - println!("Task error during shutdown: {:?}", e); - } + let _ = handle.await; - println!("Graceful shutdown complete"); + info!("Graceful shutdown complete"); } diff --git a/packages/ciphernode/evm/src/enclave_sol_writer.rs b/packages/ciphernode/evm/src/enclave_sol_writer.rs index f7444240..03cd2673 100644 --- a/packages/ciphernode/evm/src/enclave_sol_writer.rs +++ b/packages/ciphernode/evm/src/enclave_sol_writer.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use crate::helpers::create_provider_with_signer; +use crate::helpers::ensure_http_rpc; use crate::helpers::SignerProvider; use actix::prelude::*; use actix::Addr; @@ -36,7 +37,7 @@ impl EnclaveSolWriter { signer: Arc, ) -> Result { Ok(Self { - provider: create_provider_with_signer(rpc_url, signer).await?, + provider: create_provider_with_signer(&ensure_http_rpc(rpc_url), signer).await?, contract_address, bus, }) diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index 7ce67710..ae607f01 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -1,4 +1,4 @@ -use crate::helpers::{self, create_readonly_provider, ReadonlyProvider}; +use crate::helpers::{self, create_readonly_provider, ensure_ws_rpc, ReadonlyProvider}; use actix::prelude::*; use actix::{Addr, Recipient}; use alloy::primitives::{LogData, B256}; @@ -14,7 +14,7 @@ pub type ExtractorFn = fn(&LogData, Option<&B256>, u64) -> Option; /// Connects to Enclave.sol converting EVM events to EnclaveEvents pub struct EvmEventReader { - provider: ReadonlyProvider, + provider: Option, contract_address: Address, bus: Recipient, extractor: fn(&LogData, Option<&B256>, u64) -> Option, @@ -30,10 +30,10 @@ impl EvmEventReader { contract_address: Address, ) -> Result { let (shutdown_tx, shutdown_rx) = oneshot::channel(); - let provider = create_readonly_provider(rpc_url).await?; + let provider = create_readonly_provider(&ensure_ws_rpc(rpc_url)).await?; Ok(Self { contract_address, - provider, + provider: Some(provider), extractor, bus: bus.into(), shutdown_rx: Some(shutdown_rx), @@ -63,7 +63,10 @@ impl Actor for EvmEventReader { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { let bus = self.bus.clone(); - let provider = self.provider.clone(); + let Some(provider) = self.provider.take() else { + tracing::error!("Could not start event reader as provider has already been used."); + return; + }; let filter = Filter::new() .address(self.contract_address) .from_block(BlockNumberOrTag::Latest); @@ -78,14 +81,6 @@ impl Actor for EvmEventReader { .into_actor(self), ); } - - fn stopping(&mut self, _: &mut Self::Context) -> Running { - if let Some(shutdown) = self.shutdown_tx.take() { - let _ = shutdown.send(()); - } - - Running::Stop - } } impl Handler for EvmEventReader { diff --git a/packages/ciphernode/evm/src/helpers.rs b/packages/ciphernode/evm/src/helpers.rs index a727347f..dce2a077 100644 --- a/packages/ciphernode/evm/src/helpers.rs +++ b/packages/ciphernode/evm/src/helpers.rs @@ -60,7 +60,8 @@ pub async fn stream_from_evm( Err(e) => { bus.err(EnclaveErrorType::Evm, e); } - } + }; + info!("Exiting stream loop"); } #[derive(Clone)] @@ -126,6 +127,7 @@ pub async fn create_provider_with_signer( .wallet(wallet) .on_builtin(rpc_url) .await?; + Ok(SignerProvider::new(provider).await?) } @@ -135,3 +137,41 @@ pub async fn pull_eth_signer_from_env(var: &str) -> Result env::remove_var(var); Ok(Arc::new(signer)) } + +pub fn ensure_http_rpc(rpc_url: &str) -> String { + if rpc_url.starts_with("ws://") { + return rpc_url.replacen("ws://", "http://", 1); + } else if rpc_url.starts_with("wss://") { + return rpc_url.replacen("wss://", "https://", 1); + } + rpc_url.to_string() +} + +pub fn ensure_ws_rpc(rpc_url: &str) -> String { + if rpc_url.starts_with("http://") { + return rpc_url.replacen("http://", "ws://", 1); + } else if rpc_url.starts_with("https://") { + return rpc_url.replacen("https://", "wss://", 1); + } + rpc_url.to_string() +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_ensure_http_rpc() { + assert_eq!(ensure_http_rpc("http://foo.com"), "http://foo.com"); + assert_eq!(ensure_http_rpc("https://foo.com"), "https://foo.com"); + assert_eq!(ensure_http_rpc("ws://foo.com"), "http://foo.com"); + assert_eq!(ensure_http_rpc("wss://foo.com"), "https://foo.com"); + } + #[test] + fn test_ensure_ws_rpc() { + assert_eq!(ensure_ws_rpc("http://foo.com"), "ws://foo.com"); + assert_eq!(ensure_ws_rpc("https://foo.com"), "wss://foo.com"); + assert_eq!(ensure_ws_rpc("wss://foo.com"), "wss://foo.com"); + assert_eq!(ensure_ws_rpc("ws://foo.com"), "ws://foo.com"); + } +} diff --git a/packages/ciphernode/evm/src/registry_filter_sol.rs b/packages/ciphernode/evm/src/registry_filter_sol.rs index 318e065e..8684f307 100644 --- a/packages/ciphernode/evm/src/registry_filter_sol.rs +++ b/packages/ciphernode/evm/src/registry_filter_sol.rs @@ -1,4 +1,4 @@ -use crate::helpers::{create_provider_with_signer, SignerProvider}; +use crate::helpers::{create_provider_with_signer, ensure_http_rpc, SignerProvider}; use actix::prelude::*; use alloy::{ primitives::{Address, Bytes, U256}, @@ -34,7 +34,7 @@ impl RegistryFilterSolWriter { signer: Arc, ) -> Result { Ok(Self { - provider: create_provider_with_signer(rpc_url, signer).await?, + provider: create_provider_with_signer(&ensure_http_rpc(rpc_url), signer).await?, contract_address, bus, }) diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index a3896f26..1f879dc7 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -46,7 +46,7 @@ async fn setup_local_ciphernode( ) -> Result { // create data actor for saving data let data_actor = data.unwrap_or_else(|| InMemStore::new(logging).start()); - let store = DataStore::from_in_mem(&data_actor); + let store = DataStore::from(&data_actor); let repositories = store.repositories(); // create ciphernode actor for managing ciphernode flow diff --git a/tests/basic_integration/test.sh b/tests/basic_integration/test.sh index 6fb671e5..57a5fdec 100755 --- a/tests/basic_integration/test.sh +++ b/tests/basic_integration/test.sh @@ -177,5 +177,10 @@ fi heading "Test PASSED !" +pkill -15 -f "target/debug/enclave" || true +pkill -15 -f "target/debug/aggregator" || true + +sleep 4 + cleanup 0 From 59f5cb7ba1e7c01b9abe46db9cc22c312bf8c29b Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 11:42:45 +1100 Subject: [PATCH 29/49] Tidy up shutdown --- .../ciphernode/evm/src/enclave_sol_writer.rs | 19 +++++++++++++++---- .../ciphernode/evm/src/registry_filter_sol.rs | 13 ++++++++++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/packages/ciphernode/evm/src/enclave_sol_writer.rs b/packages/ciphernode/evm/src/enclave_sol_writer.rs index 03cd2673..410efa62 100644 --- a/packages/ciphernode/evm/src/enclave_sol_writer.rs +++ b/packages/ciphernode/evm/src/enclave_sol_writer.rs @@ -12,6 +12,7 @@ use alloy::{ rpc::types::TransactionReceipt, }; use anyhow::Result; +use enclave_core::Shutdown; use enclave_core::{BusError, E3id, EnclaveErrorType, PlaintextAggregated, Subscribe}; use enclave_core::{EnclaveEvent, EventBus}; use tracing::info; @@ -52,9 +53,11 @@ impl EnclaveSolWriter { let addr = EnclaveSolWriter::new(bus.clone(), rpc_url, contract_address.parse()?, signer) .await? .start(); - let _ = bus - .send(Subscribe::new("PlaintextAggregated", addr.clone().into())) - .await; + bus.send(Subscribe::new("PlaintextAggregated", addr.clone().into())) + .await?; + + bus.send(Subscribe::new("Shutdown", addr.clone().into())) + .await?; Ok(addr) } @@ -73,7 +76,8 @@ impl Handler for EnclaveSolWriter { if self.provider.get_chain_id() == data.src_chain_id { ctx.notify(data); } - } + }, + EnclaveEvent::Shutdown { data, .. } => ctx.notify(data), _ => (), } } @@ -104,6 +108,13 @@ impl Handler for EnclaveSolWriter { } } +impl Handler for EnclaveSolWriter { + type Result = (); + fn handle(&mut self, _: Shutdown, ctx: &mut Self::Context) -> Self::Result { + ctx.stop(); + } +} + async fn publish_plaintext_output( provider: SignerProvider, contract_address: Address, diff --git a/packages/ciphernode/evm/src/registry_filter_sol.rs b/packages/ciphernode/evm/src/registry_filter_sol.rs index 8684f307..bf3842be 100644 --- a/packages/ciphernode/evm/src/registry_filter_sol.rs +++ b/packages/ciphernode/evm/src/registry_filter_sol.rs @@ -8,8 +8,7 @@ use alloy::{ }; use anyhow::Result; use enclave_core::{ - BusError, E3id, EnclaveErrorType, EnclaveEvent, EventBus, OrderedSet, PublicKeyAggregated, - Subscribe, + BusError, E3id, EnclaveErrorType, EnclaveEvent, EventBus, OrderedSet, PublicKeyAggregated, Shutdown, Subscribe }; use std::sync::Arc; use tracing::info; @@ -71,7 +70,8 @@ impl Handler for RegistryFilterSolWriter { if self.provider.get_chain_id() == data.src_chain_id { ctx.notify(data); } - } + }, + EnclaveEvent::Shutdown { data, .. } => ctx.notify(data), _ => (), } } @@ -102,6 +102,13 @@ impl Handler for RegistryFilterSolWriter { } } +impl Handler for RegistryFilterSolWriter { + type Result = (); + fn handle(&mut self, _: Shutdown, ctx: &mut Self::Context) -> Self::Result { + ctx.stop(); + } +} + pub async fn publish_committee( provider: SignerProvider, contract_address: Address, From 655da02e2112126a86519ba66023b57c144e68e8 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 11:43:52 +1100 Subject: [PATCH 30/49] Formatting --- packages/ciphernode/enclave_node/src/ciphernode.rs | 8 +++++++- packages/ciphernode/enclave_node/src/lib.rs | 2 +- packages/ciphernode/evm/src/enclave_sol_writer.rs | 2 +- packages/ciphernode/evm/src/lib.rs | 4 ++-- packages/ciphernode/evm/src/registry_filter_sol.rs | 5 +++-- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index ba329dc2..2b1367be 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -100,7 +100,13 @@ impl MainCiphernode { let nm = format!("CIPHER({})", &address.to_string()[0..5]); SimpleLogger::attach(&nm, bus.clone()); MainCiphernode::new( - address, bus.clone(), store, sortition, selector, p2p_addr, e3_manager, + address, + bus.clone(), + store, + sortition, + selector, + p2p_addr, + e3_manager, ) .start(); Ok((bus, join_handle)) diff --git a/packages/ciphernode/enclave_node/src/lib.rs b/packages/ciphernode/enclave_node/src/lib.rs index 7f475274..533fe309 100644 --- a/packages/ciphernode/enclave_node/src/lib.rs +++ b/packages/ciphernode/enclave_node/src/lib.rs @@ -4,6 +4,6 @@ mod ciphernode; mod shutdown; pub use aggregator::*; -pub use ciphernode::*; pub use app_config::*; +pub use ciphernode::*; pub use shutdown::*; diff --git a/packages/ciphernode/evm/src/enclave_sol_writer.rs b/packages/ciphernode/evm/src/enclave_sol_writer.rs index 410efa62..5c8fcf10 100644 --- a/packages/ciphernode/evm/src/enclave_sol_writer.rs +++ b/packages/ciphernode/evm/src/enclave_sol_writer.rs @@ -76,7 +76,7 @@ impl Handler for EnclaveSolWriter { if self.provider.get_chain_id() == data.src_chain_id { ctx.notify(data); } - }, + } EnclaveEvent::Shutdown { data, .. } => ctx.notify(data), _ => (), } diff --git a/packages/ciphernode/evm/src/lib.rs b/packages/ciphernode/evm/src/lib.rs index 51d70288..d13e040a 100644 --- a/packages/ciphernode/evm/src/lib.rs +++ b/packages/ciphernode/evm/src/lib.rs @@ -2,13 +2,13 @@ mod ciphernode_registry_sol; mod enclave_sol; mod enclave_sol_reader; mod enclave_sol_writer; +mod event_reader; pub mod helpers; mod registry_filter_sol; -mod event_reader; pub use ciphernode_registry_sol::{CiphernodeRegistrySol, CiphernodeRegistrySolReader}; pub use enclave_sol::EnclaveSol; pub use enclave_sol_reader::EnclaveSolReader; pub use enclave_sol_writer::EnclaveSolWriter; -pub use registry_filter_sol::{RegistryFilterSol, RegistryFilterSolWriter}; pub use event_reader::EvmEventReader; +pub use registry_filter_sol::{RegistryFilterSol, RegistryFilterSolWriter}; diff --git a/packages/ciphernode/evm/src/registry_filter_sol.rs b/packages/ciphernode/evm/src/registry_filter_sol.rs index bf3842be..244e204f 100644 --- a/packages/ciphernode/evm/src/registry_filter_sol.rs +++ b/packages/ciphernode/evm/src/registry_filter_sol.rs @@ -8,7 +8,8 @@ use alloy::{ }; use anyhow::Result; use enclave_core::{ - BusError, E3id, EnclaveErrorType, EnclaveEvent, EventBus, OrderedSet, PublicKeyAggregated, Shutdown, Subscribe + BusError, E3id, EnclaveErrorType, EnclaveEvent, EventBus, OrderedSet, PublicKeyAggregated, + Shutdown, Subscribe, }; use std::sync::Arc; use tracing::info; @@ -70,7 +71,7 @@ impl Handler for RegistryFilterSolWriter { if self.provider.get_chain_id() == data.src_chain_id { ctx.notify(data); } - }, + } EnclaveEvent::Shutdown { data, .. } => ctx.notify(data), _ => (), } From 8dadf56add1c757861e4c3e55f9f0d62497be9ba Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 13:16:53 +1100 Subject: [PATCH 31/49] Move clone down the into stack --- packages/ciphernode/data/src/repository.rs | 6 ++++++ packages/ciphernode/router/src/repositories.rs | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/ciphernode/data/src/repository.rs b/packages/ciphernode/data/src/repository.rs index e1c6d803..8fd570ff 100644 --- a/packages/ciphernode/data/src/repository.rs +++ b/packages/ciphernode/data/src/repository.rs @@ -31,6 +31,12 @@ impl From> for DataStore { } } +impl From<&Repository> for DataStore { + fn from(value: &Repository) -> Self { + value.store.clone() + } +} + /// Clone without phantom data impl Clone for Repository { fn clone(&self) -> Self { diff --git a/packages/ciphernode/router/src/repositories.rs b/packages/ciphernode/router/src/repositories.rs index 1d93c179..ed1a5b37 100644 --- a/packages/ciphernode/router/src/repositories.rs +++ b/packages/ciphernode/router/src/repositories.rs @@ -82,7 +82,9 @@ impl RepositoriesFactory for DataStore { impl RepositoriesFactory for Repository { fn repositories(&self) -> Repositories { - let store: DataStore = self.clone().into(); + let store: DataStore = self.into(); store.repositories() } } + + From 5477111a0d42fdb586d2aa4399f5a39841bb5135 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 13:19:54 +1100 Subject: [PATCH 32/49] Format --- packages/ciphernode/data/src/repository.rs | 2 +- packages/ciphernode/router/src/repositories.rs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/ciphernode/data/src/repository.rs b/packages/ciphernode/data/src/repository.rs index 8fd570ff..7cf1e445 100644 --- a/packages/ciphernode/data/src/repository.rs +++ b/packages/ciphernode/data/src/repository.rs @@ -33,7 +33,7 @@ impl From> for DataStore { impl From<&Repository> for DataStore { fn from(value: &Repository) -> Self { - value.store.clone() + value.store.clone() } } diff --git a/packages/ciphernode/router/src/repositories.rs b/packages/ciphernode/router/src/repositories.rs index ed1a5b37..4a155da8 100644 --- a/packages/ciphernode/router/src/repositories.rs +++ b/packages/ciphernode/router/src/repositories.rs @@ -86,5 +86,3 @@ impl RepositoriesFactory for Repository { store.repositories() } } - - From 13727942c8c65866fa3792ec296b2593be9b7847 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 13:21:34 +1100 Subject: [PATCH 33/49] Remove comment --- .../ciphernode/tests/tests/test_aggregation_and_decryption.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 1f879dc7..168248f9 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -523,9 +523,6 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { Ok(()) } -// #[actix::test] -// async fn test_actors_resume_from_shutdown() -> Result<()> {} - #[actix::test] async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { let seed = Seed(ChaCha20Rng::seed_from_u64(123).get_seed()); From e1c3394935273dc596b3aea01993cc3a342054d1 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 13:23:02 +1100 Subject: [PATCH 34/49] Formatting --- packages/ciphernode/router/src/hooks.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index 7fbcc831..14825ca7 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -166,13 +166,15 @@ impl E3Feature for PlaintextAggregatorFeature { let EnclaveEvent::CiphertextOutputPublished { data, .. } = evt else { return; }; + let Some(fhe) = ctx.get_fhe() else { - self. bus.err(EnclaveErrorType::PlaintextAggregation, anyhow!("Could not create PlaintextAggregator because the fhe instance it depends on was not set on the context.")); + self.bus.err(EnclaveErrorType::PlaintextAggregation, anyhow!("Could not create PlaintextAggregator because the fhe instance it depends on was not set on the context.")); return; }; + let Some(ref meta) = ctx.get_meta() else { - self. bus.err(EnclaveErrorType::PlaintextAggregation, anyhow!("Could not create PlaintextAggregator because the meta instance it depends on was not set on the context.")); + self.bus.err(EnclaveErrorType::PlaintextAggregation, anyhow!("Could not create PlaintextAggregator because the meta instance it depends on was not set on the context.")); return; }; From 15b414f29349c5816733689f919032c762fc7ff6 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 13:29:32 +1100 Subject: [PATCH 35/49] Log error --- packages/ciphernode/Cargo.lock | 1 + packages/ciphernode/data/Cargo.toml | 1 + packages/ciphernode/data/src/data_store.rs | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 1d6499c5..71aeb552 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -1870,6 +1870,7 @@ dependencies = [ "enclave-core", "serde", "sled", + "tracing", ] [[package]] diff --git a/packages/ciphernode/data/Cargo.toml b/packages/ciphernode/data/Cargo.toml index e1b13cc8..e3aeba38 100644 --- a/packages/ciphernode/data/Cargo.toml +++ b/packages/ciphernode/data/Cargo.toml @@ -12,6 +12,7 @@ anyhow = { workspace = true } serde = { workspace = true } sled = { workspace = true } bincode = { workspace = true } +tracing = { workspace = true } async-trait = { workspace = true } [dev-dependencies] diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index 7028b090..0c03266d 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -2,6 +2,7 @@ use crate::{InMemStore, IntoKey, SledStore}; use actix::{Addr, Message, Recipient}; use anyhow::Result; use serde::{Deserialize, Serialize}; +use tracing::error; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "()")] @@ -57,6 +58,9 @@ impl DataStore { /// Writes data to the scope location pub fn write(&self, value: T) { let Ok(serialized) = bincode::serialize(&value) else { + let str_key = self.get_scope().unwrap_or("".to_string()); + let str_error = format!("Could not serialize value passed to {}", str_key); + error!(str_error); return; }; let msg = Insert::new(&self.scope, serialized); From 157423bda4fef956a05629bab024cbd703bdbc81 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 13:34:05 +1100 Subject: [PATCH 36/49] Avoid cloning --- packages/ciphernode/data/src/data_store.rs | 12 ++++++------ packages/ciphernode/data/src/in_mem.rs | 4 ++-- packages/ciphernode/data/src/sled_store.rs | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index 0c03266d..84dede64 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -12,12 +12,12 @@ impl Insert { Self(key.into_key(), value) } - pub fn key(&self) -> Vec { - self.0.clone() + pub fn key(&self) -> &Vec { + &self.0 } - pub fn value(&self) -> Vec { - self.1.clone() + pub fn value(&self) -> &Vec { + &self.1 } } @@ -29,8 +29,8 @@ impl Get { Self(key.into_key()) } - pub fn key(&self) -> Vec { - self.0.clone() + pub fn key(&self) -> &Vec { + &self.0 } } diff --git a/packages/ciphernode/data/src/in_mem.rs b/packages/ciphernode/data/src/in_mem.rs index 6821de72..b3909d52 100644 --- a/packages/ciphernode/data/src/in_mem.rs +++ b/packages/ciphernode/data/src/in_mem.rs @@ -36,7 +36,7 @@ impl Handler for InMemStore { type Result = (); fn handle(&mut self, event: Insert, _: &mut Self::Context) { // insert data into sled - self.db.insert(event.key(), event.value()); + self.db.insert(event.key().to_vec(), event.value().to_vec()); if self.capture { self.log.push(DataOp::Insert(event)); @@ -48,7 +48,7 @@ impl Handler for InMemStore { type Result = Option>; fn handle(&mut self, event: Get, _: &mut Self::Context) -> Option> { let key = event.key(); - self.db.get(&key).map(|r| r.clone()) + self.db.get(key).cloned() } } diff --git a/packages/ciphernode/data/src/sled_store.rs b/packages/ciphernode/data/src/sled_store.rs index cfe1ead4..e87eec9d 100644 --- a/packages/ciphernode/data/src/sled_store.rs +++ b/packages/ciphernode/data/src/sled_store.rs @@ -29,7 +29,7 @@ impl Handler for SledStore { fn handle(&mut self, event: Insert, _: &mut Self::Context) -> Self::Result { match self .db - .insert(event.key(), event.value()) + .insert(event.key(), event.value().to_vec()) .context("Could not insert data into db") { Err(err) => self.bus.err(EnclaveErrorType::Data, err), From c93e962d558866a9182920a5ced72e01955f33d6 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 13:50:02 +1100 Subject: [PATCH 37/49] Add strings as constants --- packages/ciphernode/router/src/hooks.rs | 62 +++++++++++++++++++++---- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index 14825ca7..ff9950b7 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -81,6 +81,9 @@ impl KeyshareFeature { } } +const ERROR_KEYSHARE_FHE_MISSING: &str = + "Could not create Keyshare because the fhe instance it depends on was not set on the context."; + #[async_trait] impl E3Feature for KeyshareFeature { fn on_event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { @@ -90,7 +93,10 @@ impl E3Feature for KeyshareFeature { }; let Some(fhe) = ctx.get_fhe() else { - self.bus.err(EnclaveErrorType::KeyGeneration, anyhow!("Could not create Keyshare because the fhe instance it depends on was not set on the context.")); + self.bus.err( + EnclaveErrorType::KeyGeneration, + anyhow!(ERROR_KEYSHARE_FHE_MISSING), + ); return; }; @@ -126,6 +132,10 @@ impl E3Feature for KeyshareFeature { // Get deps let Some(fhe) = ctx.fhe.clone() else { + self.bus.err( + EnclaveErrorType::KeyGeneration, + anyhow!(ERROR_KEYSHARE_FHE_MISSING), + ); return Ok(()); }; @@ -159,6 +169,9 @@ impl PlaintextAggregatorFeature { } } +const ERROR_PLAINTEXT_FHE_MISSING:&str = "Could not create PlaintextAggregator because the fhe instance it depends on was not set on the context."; +const ERROR_PLAINTEXT_META_MISSING:&str = "Could not create PlaintextAggregator because the meta instance it depends on was not set on the context."; + #[async_trait] impl E3Feature for PlaintextAggregatorFeature { fn on_event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { @@ -168,14 +181,18 @@ impl E3Feature for PlaintextAggregatorFeature { }; let Some(fhe) = ctx.get_fhe() else { - self.bus.err(EnclaveErrorType::PlaintextAggregation, anyhow!("Could not create PlaintextAggregator because the fhe instance it depends on was not set on the context.")); - + self.bus.err( + EnclaveErrorType::PlaintextAggregation, + anyhow!(ERROR_PLAINTEXT_FHE_MISSING), + ); return; }; let Some(ref meta) = ctx.get_meta() else { - self.bus.err(EnclaveErrorType::PlaintextAggregation, anyhow!("Could not create PlaintextAggregator because the meta instance it depends on was not set on the context.")); - + self.bus.err( + EnclaveErrorType::PlaintextAggregation, + anyhow!(ERROR_PLAINTEXT_META_MISSING), + ); return; }; @@ -219,11 +236,19 @@ impl E3Feature for PlaintextAggregatorFeature { }; // Get deps - let Some(fhe) = ctx.fhe.clone() else { + let Some(fhe) = ctx.get_fhe() else { + self.bus.err( + EnclaveErrorType::PlaintextAggregation, + anyhow!(ERROR_PLAINTEXT_FHE_MISSING), + ); return Ok(()); }; - let Some(meta) = ctx.meta.clone() else { + let Some(ref meta) = ctx.get_meta() else { + self.bus.err( + EnclaveErrorType::PlaintextAggregation, + anyhow!(ERROR_PLAINTEXT_META_MISSING), + ); return Ok(()); }; @@ -259,6 +284,9 @@ impl PublicKeyAggregatorFeature { } } +const ERROR_PUBKEY_FHE_MISSING:&str = "Could not create PublicKeyAggregator because the fhe instance it depends on was not set on the context."; +const ERROR_PUBKEY_META_MISSING:&str = "Could not create PublicKeyAggregator because the meta instance it depends on was not set on the context."; + #[async_trait] impl E3Feature for PublicKeyAggregatorFeature { fn on_event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent) { @@ -268,11 +296,17 @@ impl E3Feature for PublicKeyAggregatorFeature { }; let Some(fhe) = ctx.get_fhe() else { - self.bus.err(EnclaveErrorType::PublickeyAggregation, anyhow!("Could not create PublicKeyAggregator because the fhe instance it depends on was not set on the context.")); + self.bus.err( + EnclaveErrorType::PublickeyAggregation, + anyhow!(ERROR_PUBKEY_FHE_MISSING), + ); return; }; let Some(ref meta) = ctx.get_meta() else { - self.bus.err(EnclaveErrorType::PublickeyAggregation, anyhow!("Could not create PublicKeyAggregator because the meta instance it depends on was not set on the context.")); + self.bus.err( + EnclaveErrorType::PublickeyAggregation, + anyhow!(ERROR_PUBKEY_META_MISSING), + ); return; }; @@ -313,10 +347,20 @@ impl E3Feature for PublicKeyAggregatorFeature { // Get deps let Some(fhe) = ctx.fhe.clone() else { + self.bus.err( + EnclaveErrorType::PublickeyAggregation, + anyhow!(ERROR_PUBKEY_FHE_MISSING), + ); + return Ok(()); }; let Some(meta) = ctx.meta.clone() else { + self.bus.err( + EnclaveErrorType::PublickeyAggregation, + anyhow!(ERROR_PUBKEY_META_MISSING), + ); + return Ok(()); }; From 24881bba7042cc7fe357f7dca8039f2cedb693ba Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 14:12:19 +1100 Subject: [PATCH 38/49] Sort out strings and references --- .../ciphernode/enclave_node/src/aggregator.rs | 41 ++++------------ .../ciphernode/enclave_node/src/ciphernode.rs | 21 ++++----- .../evm/src/ciphernode_registry_sol.rs | 6 +-- packages/ciphernode/evm/src/enclave_sol.rs | 6 +-- .../ciphernode/evm/src/enclave_sol_reader.rs | 2 +- .../ciphernode/evm/src/enclave_sol_writer.rs | 12 ++--- packages/ciphernode/evm/src/event_reader.rs | 8 ++-- packages/ciphernode/evm/src/helpers.rs | 2 +- .../ciphernode/evm/src/registry_filter_sol.rs | 23 +++++---- .../router/src/ciphernode_selector.rs | 10 ++-- .../router/src/e3_request_router.rs | 4 +- packages/ciphernode/router/src/hooks.rs | 47 ++++++++++++++----- .../ciphernode/sortition/src/sortition.rs | 2 +- .../tests/test_aggregation_and_decryption.rs | 22 ++++----- 14 files changed, 98 insertions(+), 108 deletions(-) diff --git a/packages/ciphernode/enclave_node/src/aggregator.rs b/packages/ciphernode/enclave_node/src/aggregator.rs index 938c409f..979115f5 100644 --- a/packages/ciphernode/enclave_node/src/aggregator.rs +++ b/packages/ciphernode/enclave_node/src/aggregator.rs @@ -62,7 +62,7 @@ impl MainAggregator { }; let repositories = store.repositories(); - let sortition = Sortition::attach(bus.clone(), repositories.sortition()); + let sortition = Sortition::attach(&bus, repositories.sortition()); let signer = pull_eth_signer_from_env("PRIVATE_KEY").await?; for chain in config .chains @@ -70,38 +70,17 @@ impl MainAggregator { .filter(|chain| chain.enabled.unwrap_or(true)) { let rpc_url = &chain.rpc_url; - EnclaveSol::attach( - bus.clone(), - rpc_url, - &chain.contracts.enclave, - signer.clone(), - ) - .await?; - RegistryFilterSol::attach( - bus.clone(), - rpc_url, - &chain.contracts.filter_registry, - signer.clone(), - ) - .await?; - CiphernodeRegistrySol::attach( - bus.clone(), - rpc_url, - &chain.contracts.ciphernode_registry, - ) - .await?; + EnclaveSol::attach(&bus, rpc_url, &chain.contracts.enclave, &signer).await?; + RegistryFilterSol::attach(&bus, rpc_url, &chain.contracts.filter_registry, &signer) + .await?; + CiphernodeRegistrySol::attach(&bus, rpc_url, &chain.contracts.ciphernode_registry) + .await?; } - let e3_manager = E3RequestRouter::builder(bus.clone(), store) - .add_feature(FheFeature::create(rng)) - .add_feature(PublicKeyAggregatorFeature::create( - bus.clone(), - sortition.clone(), - )) - .add_feature(PlaintextAggregatorFeature::create( - bus.clone(), - sortition.clone(), - )) + let e3_manager = E3RequestRouter::builder(&bus, store) + .add_feature(FheFeature::create(&bus, &rng)) + .add_feature(PublicKeyAggregatorFeature::create(&bus, &sortition)) + .add_feature(PlaintextAggregatorFeature::create(&bus, &sortition)) .build() .await?; diff --git a/packages/ciphernode/enclave_node/src/ciphernode.rs b/packages/ciphernode/enclave_node/src/ciphernode.rs index 2b1367be..21ca449e 100644 --- a/packages/ciphernode/enclave_node/src/ciphernode.rs +++ b/packages/ciphernode/enclave_node/src/ciphernode.rs @@ -68,9 +68,8 @@ impl MainCiphernode { let repositories = store.repositories(); - let sortition = Sortition::attach(bus.clone(), repositories.sortition()); - let selector = - CiphernodeSelector::attach(bus.clone(), sortition.clone(), &address.to_string()); + let sortition = Sortition::attach(&bus, repositories.sortition()); + let selector = CiphernodeSelector::attach(&bus, &sortition, &address.to_string()); for chain in config .chains @@ -79,18 +78,14 @@ impl MainCiphernode { { let rpc_url = &chain.rpc_url; - EnclaveSolReader::attach(bus.clone(), rpc_url, &chain.contracts.enclave).await?; - CiphernodeRegistrySol::attach( - bus.clone(), - rpc_url, - &chain.contracts.ciphernode_registry, - ) - .await?; + EnclaveSolReader::attach(&bus, rpc_url, &chain.contracts.enclave).await?; + CiphernodeRegistrySol::attach(&bus, rpc_url, &chain.contracts.ciphernode_registry) + .await?; } - let e3_manager = E3RequestRouter::builder(bus.clone(), store.clone()) - .add_feature(FheFeature::create(rng)) - .add_feature(KeyshareFeature::create(bus.clone(), &address.to_string())) + let e3_manager = E3RequestRouter::builder(&bus, store.clone()) + .add_feature(FheFeature::create(&bus, &rng)) + .add_feature(KeyshareFeature::create(&bus, &address.to_string())) .build() .await?; diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index f4f4b7b3..2fd56359 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -96,7 +96,7 @@ pub struct CiphernodeRegistrySolReader; impl CiphernodeRegistrySolReader { pub async fn attach( - bus: Addr, + bus: &Addr, rpc_url: &str, contract_address: &str, ) -> Result> { @@ -107,8 +107,8 @@ impl CiphernodeRegistrySolReader { pub struct CiphernodeRegistrySol; impl CiphernodeRegistrySol { - pub async fn attach(bus: Addr, rpc_url: &str, contract_address: &str) -> Result<()> { - CiphernodeRegistrySolReader::attach(bus.clone(), rpc_url, contract_address).await?; + pub async fn attach(bus: &Addr, rpc_url: &str, contract_address: &str) -> Result<()> { + CiphernodeRegistrySolReader::attach(bus, rpc_url, contract_address).await?; Ok(()) } } diff --git a/packages/ciphernode/evm/src/enclave_sol.rs b/packages/ciphernode/evm/src/enclave_sol.rs index ba1631c0..12c67f90 100644 --- a/packages/ciphernode/evm/src/enclave_sol.rs +++ b/packages/ciphernode/evm/src/enclave_sol.rs @@ -9,12 +9,12 @@ use enclave_core::EventBus; pub struct EnclaveSol; impl EnclaveSol { pub async fn attach( - bus: Addr, + bus: &Addr, rpc_url: &str, contract_address: &str, - signer: Arc, + signer: &Arc, ) -> Result<()> { - EnclaveSolReader::attach(bus.clone(), rpc_url, contract_address).await?; + EnclaveSolReader::attach(bus, rpc_url, contract_address).await?; EnclaveSolWriter::attach(bus, rpc_url, contract_address, signer).await?; Ok(()) } diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index e46c618c..46c061d1 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -81,7 +81,7 @@ pub struct EnclaveSolReader; impl EnclaveSolReader { pub async fn attach( - bus: Addr, + bus: &Addr, rpc_url: &str, contract_address: &str, ) -> Result> { diff --git a/packages/ciphernode/evm/src/enclave_sol_writer.rs b/packages/ciphernode/evm/src/enclave_sol_writer.rs index 5c8fcf10..1c99277c 100644 --- a/packages/ciphernode/evm/src/enclave_sol_writer.rs +++ b/packages/ciphernode/evm/src/enclave_sol_writer.rs @@ -32,25 +32,25 @@ pub struct EnclaveSolWriter { impl EnclaveSolWriter { pub async fn new( - bus: Addr, + bus: &Addr, rpc_url: &str, contract_address: Address, - signer: Arc, + signer: &Arc, ) -> Result { Ok(Self { provider: create_provider_with_signer(&ensure_http_rpc(rpc_url), signer).await?, contract_address, - bus, + bus: bus.clone(), }) } pub async fn attach( - bus: Addr, + bus: &Addr, rpc_url: &str, contract_address: &str, - signer: Arc, + signer: &Arc, ) -> Result> { - let addr = EnclaveSolWriter::new(bus.clone(), rpc_url, contract_address.parse()?, signer) + let addr = EnclaveSolWriter::new(bus, rpc_url, contract_address.parse()?, signer) .await? .start(); bus.send(Subscribe::new("PlaintextAggregated", addr.clone().into())) diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index ae607f01..c64dd775 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -24,7 +24,7 @@ pub struct EvmEventReader { impl EvmEventReader { pub async fn new( - bus: Addr, + bus: &Addr, rpc_url: &str, extractor: ExtractorFn, contract_address: Address, @@ -35,19 +35,19 @@ impl EvmEventReader { contract_address, provider: Some(provider), extractor, - bus: bus.into(), + bus: bus.clone().into(), shutdown_rx: Some(shutdown_rx), shutdown_tx: Some(shutdown_tx), }) } pub async fn attach( - bus: Addr, + bus: &Addr, rpc_url: &str, extractor: ExtractorFn, contract_address: &str, ) -> Result> { - let addr = EvmEventReader::new(bus.clone(), rpc_url, extractor, contract_address.parse()?) + let addr = EvmEventReader::new(bus, rpc_url, extractor, contract_address.parse()?) .await? .start(); diff --git a/packages/ciphernode/evm/src/helpers.rs b/packages/ciphernode/evm/src/helpers.rs index dce2a077..4aa15dca 100644 --- a/packages/ciphernode/evm/src/helpers.rs +++ b/packages/ciphernode/evm/src/helpers.rs @@ -119,7 +119,7 @@ pub type SignerProvider = WithChainId< pub async fn create_provider_with_signer( rpc_url: &str, - signer: Arc, + signer: &Arc, ) -> Result { let wallet = EthereumWallet::from(signer.clone()); let provider = ProviderBuilder::new() diff --git a/packages/ciphernode/evm/src/registry_filter_sol.rs b/packages/ciphernode/evm/src/registry_filter_sol.rs index 244e204f..2c64129a 100644 --- a/packages/ciphernode/evm/src/registry_filter_sol.rs +++ b/packages/ciphernode/evm/src/registry_filter_sol.rs @@ -28,28 +28,27 @@ pub struct RegistryFilterSolWriter { impl RegistryFilterSolWriter { pub async fn new( - bus: Addr, + bus: &Addr, rpc_url: &str, contract_address: Address, - signer: Arc, + signer: &Arc, ) -> Result { Ok(Self { provider: create_provider_with_signer(&ensure_http_rpc(rpc_url), signer).await?, contract_address, - bus, + bus: bus.clone(), }) } pub async fn attach( - bus: Addr, + bus: &Addr, rpc_url: &str, contract_address: &str, - signer: Arc, + signer: &Arc, ) -> Result> { - let addr = - RegistryFilterSolWriter::new(bus.clone(), rpc_url, contract_address.parse()?, signer) - .await? - .start(); + let addr = RegistryFilterSolWriter::new(bus, rpc_url, contract_address.parse()?, signer) + .await? + .start(); let _ = bus .send(Subscribe::new("PublicKeyAggregated", addr.clone().into())) .await; @@ -132,12 +131,12 @@ pub async fn publish_committee( pub struct RegistryFilterSol; impl RegistryFilterSol { pub async fn attach( - bus: Addr, + bus: &Addr, rpc_url: &str, contract_address: &str, - signer: Arc, + signer: &Arc, ) -> Result<()> { - RegistryFilterSolWriter::attach(bus.clone(), rpc_url, contract_address, signer).await?; + RegistryFilterSolWriter::attach(bus, rpc_url, contract_address, signer).await?; Ok(()) } } diff --git a/packages/ciphernode/router/src/ciphernode_selector.rs b/packages/ciphernode/router/src/ciphernode_selector.rs index b0dbd08c..96048a67 100644 --- a/packages/ciphernode/router/src/ciphernode_selector.rs +++ b/packages/ciphernode/router/src/ciphernode_selector.rs @@ -16,16 +16,16 @@ impl Actor for CiphernodeSelector { } impl CiphernodeSelector { - pub fn new(bus: Addr, sortition: Addr, address: &str) -> Self { + pub fn new(bus: &Addr, sortition: &Addr, address: &str) -> Self { Self { - bus, - sortition, + bus: bus.clone(), + sortition: sortition.clone(), address: address.to_owned(), } } - pub fn attach(bus: Addr, sortition: Addr, address: &str) -> Addr { - let addr = CiphernodeSelector::new(bus.clone(), sortition, address).start(); + pub fn attach(bus: &Addr, sortition: &Addr, address: &str) -> Addr { + let addr = CiphernodeSelector::new(bus, sortition, address).start(); bus.do_send(Subscribe::new("E3Requested", addr.clone().recipient())); bus.do_send(Subscribe::new("Shutdown", addr.clone().recipient())); diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index 35d45c32..86684c6f 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -75,10 +75,10 @@ pub struct E3RequestRouterParams { } impl E3RequestRouter { - pub fn builder(bus: Addr, store: DataStore) -> E3RequestRouterBuilder { + pub fn builder(bus: &Addr, store: DataStore) -> E3RequestRouterBuilder { let repositories = store.repositories(); let builder = E3RequestRouterBuilder { - bus, + bus: bus.clone(), features: vec![], store: repositories.router(), }; diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index ff9950b7..4f36d799 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -11,19 +11,26 @@ use enclave_core::{BusError, E3Requested, EnclaveErrorType, EnclaveEvent, EventB use fhe::{Fhe, SharedRng}; use keyshare::{Keyshare, KeyshareParams}; use sortition::Sortition; -use std::sync::Arc; +use std::sync::{mpsc::Receiver, Arc}; +use tracing::error; /// TODO: move these to each package with access on MyStruct::launcher() pub struct FheFeature { rng: SharedRng, + bus: Addr, } impl FheFeature { - pub fn create(rng: SharedRng) -> Box { - Box::new(Self { rng }) + pub fn create(bus: &Addr, rng: &SharedRng) -> Box { + Box::new(Self { + rng: rng.clone(), + bus: bus.clone(), + }) } } +const ERROR_FHE_FAILED_TO_DECODE: &str = "Failed to decode encoded FHE params"; + #[async_trait] impl E3Feature for FheFeature { fn on_event(&self, ctx: &mut crate::E3RequestContext, evt: &EnclaveEvent) { @@ -39,9 +46,19 @@ impl E3Feature for FheFeature { .. } = data.clone(); + let Ok(fhe_inner) = Fhe::from_encoded(¶ms, seed, self.rng.clone()) else { + self.bus.err( + EnclaveErrorType::KeyGeneration, + anyhow!(ERROR_FHE_FAILED_TO_DECODE), + ); + return; + }; + + let fhe = Arc::new(fhe_inner); + // FHE doesn't implement Checkpoint so we are going to store it manually - let fhe = Arc::new(Fhe::from_encoded(¶ms, seed, self.rng.clone()).unwrap()); ctx.repositories().fhe(&e3_id).write(&fhe.snapshot()); + let _ = ctx.set_fhe(fhe); } @@ -50,12 +67,12 @@ impl E3Feature for FheFeature { ctx: &mut E3RequestContext, snapshot: &E3RequestContextSnapshot, ) -> Result<()> { - // No ID on the snapshot -> bail + // No ID on the snapshot -> bail without reporting if !snapshot.fhe { return Ok(()); }; - // No Snapshot returned from the store -> bail + // No Snapshot returned from the store -> bail without reporting let Some(snap) = ctx.repositories().fhe(&ctx.e3_id).read().await? else { return Ok(()); }; @@ -73,9 +90,9 @@ pub struct KeyshareFeature { } impl KeyshareFeature { - pub fn create(bus: Addr, address: &str) -> Box { + pub fn create(bus: &Addr, address: &str) -> Box { Box::new(Self { - bus, + bus: bus.clone(), address: address.to_owned(), }) } @@ -164,8 +181,11 @@ pub struct PlaintextAggregatorFeature { sortition: Addr, } impl PlaintextAggregatorFeature { - pub fn create(bus: Addr, sortition: Addr) -> Box { - Box::new(Self { bus, sortition }) + pub fn create(bus: &Addr, sortition: &Addr) -> Box { + Box::new(Self { + bus: bus.clone(), + sortition: sortition.clone(), + }) } } @@ -279,8 +299,11 @@ pub struct PublicKeyAggregatorFeature { } impl PublicKeyAggregatorFeature { - pub fn create(bus: Addr, sortition: Addr) -> Box { - Box::new(Self { bus, sortition }) + pub fn create(bus: &Addr, sortition: &Addr) -> Box { + Box::new(Self { + bus: bus.clone(), + sortition: sortition.clone(), + }) } } diff --git a/packages/ciphernode/sortition/src/sortition.rs b/packages/ciphernode/sortition/src/sortition.rs index 7794d088..0f992c96 100644 --- a/packages/ciphernode/sortition/src/sortition.rs +++ b/packages/ciphernode/sortition/src/sortition.rs @@ -101,7 +101,7 @@ impl Sortition { } } - pub fn attach(bus: Addr, store: Repository) -> Addr { + pub fn attach(bus: &Addr, store: Repository) -> Addr { let addr = Sortition::new(SortitionParams { bus: bus.clone(), store, diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index 168248f9..015058d2 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -50,20 +50,14 @@ async fn setup_local_ciphernode( let repositories = store.repositories(); // create ciphernode actor for managing ciphernode flow - let sortition = Sortition::attach(bus.clone(), repositories.sortition()); - CiphernodeSelector::attach(bus.clone(), sortition.clone(), addr); - - let router = E3RequestRouter::builder(bus.clone(), store) - .add_feature(FheFeature::create(rng.clone())) - .add_feature(PublicKeyAggregatorFeature::create( - bus.clone(), - sortition.clone(), - )) - .add_feature(PlaintextAggregatorFeature::create( - bus.clone(), - sortition.clone(), - )) - .add_feature(KeyshareFeature::create(bus.clone(), addr)) + let sortition = Sortition::attach(&bus, repositories.sortition()); + CiphernodeSelector::attach(&bus, &sortition, addr); + + let router = E3RequestRouter::builder(&bus, store) + .add_feature(FheFeature::create(&bus, &rng)) + .add_feature(PublicKeyAggregatorFeature::create(&bus, &sortition)) + .add_feature(PlaintextAggregatorFeature::create(&bus, &sortition)) + .add_feature(KeyshareFeature::create(&bus, addr)) .build() .await?; From 434475450cb182458f8080029e05e2369721df4e Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 14:20:03 +1100 Subject: [PATCH 39/49] Use Cow string as will be utf8 --- packages/ciphernode/data/src/data_store.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/ciphernode/data/src/data_store.rs b/packages/ciphernode/data/src/data_store.rs index 84dede64..7a15cdff 100644 --- a/packages/ciphernode/data/src/data_store.rs +++ b/packages/ciphernode/data/src/data_store.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + use crate::{InMemStore, IntoKey, SledStore}; use actix::{Addr, Message, Recipient}; use anyhow::Result; @@ -58,7 +60,7 @@ impl DataStore { /// Writes data to the scope location pub fn write(&self, value: T) { let Ok(serialized) = bincode::serialize(&value) else { - let str_key = self.get_scope().unwrap_or("".to_string()); + let str_key = self.get_scope().unwrap_or(Cow::Borrowed("")); let str_error = format!("Could not serialize value passed to {}", str_key); error!(str_error); return; @@ -68,8 +70,8 @@ impl DataStore { } /// Get the scope as a string - pub fn get_scope(&self) -> Result { - Ok(String::from_utf8(self.scope.clone())?) + pub fn get_scope(&self) -> Result> { + Ok(String::from_utf8_lossy(&self.scope)) } /// Changes the scope for the data store. From ba32ea146879e38c594682897372fda6a7b5535c Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 14:29:36 +1100 Subject: [PATCH 40/49] Add comment --- packages/ciphernode/enclave_node/src/shutdown.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ciphernode/enclave_node/src/shutdown.rs b/packages/ciphernode/enclave_node/src/shutdown.rs index ce36c4ee..a30aaab3 100644 --- a/packages/ciphernode/enclave_node/src/shutdown.rs +++ b/packages/ciphernode/enclave_node/src/shutdown.rs @@ -20,6 +20,7 @@ pub async fn listen_for_shutdown(bus: Recipient, handle: JoinHandl // Abort the spawned task handle.abort(); + // Wait for all actor processes to disconnect tokio::time::sleep(Duration::from_secs(2)).await; // Wait for the task to finish From 492604eeedc7ce3f9357ba6f1bbcca3eb99bee24 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 14:31:48 +1100 Subject: [PATCH 41/49] Remove clone --- packages/ciphernode/data/src/into_key.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ciphernode/data/src/into_key.rs b/packages/ciphernode/data/src/into_key.rs index 5da1c550..2664c7d9 100644 --- a/packages/ciphernode/data/src/into_key.rs +++ b/packages/ciphernode/data/src/into_key.rs @@ -6,7 +6,7 @@ pub trait IntoKey { /// Keys can be vectors of String impl IntoKey for Vec { fn into_key(self) -> Vec { - self.clone() + self } } From 7b0d975d2037a684b49a51b68c23ab480664b695 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 14:32:14 +1100 Subject: [PATCH 42/49] update docs --- packages/ciphernode/data/src/into_key.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ciphernode/data/src/into_key.rs b/packages/ciphernode/data/src/into_key.rs index 2664c7d9..517ff9bd 100644 --- a/packages/ciphernode/data/src/into_key.rs +++ b/packages/ciphernode/data/src/into_key.rs @@ -10,7 +10,7 @@ impl IntoKey for Vec { } } -/// Keys can be vectors of String ++/// Keys can be references to vectors of bytes (&Vec) impl IntoKey for &Vec { fn into_key(self) -> Vec { self.clone() From b53845df45f242005d3036c8cd0addac4305e417 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 14:33:48 +1100 Subject: [PATCH 43/49] Add context to sled db error --- packages/ciphernode/data/src/into_key.rs | 2 +- packages/ciphernode/data/src/sled_store.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/ciphernode/data/src/into_key.rs b/packages/ciphernode/data/src/into_key.rs index 517ff9bd..1fc7df4e 100644 --- a/packages/ciphernode/data/src/into_key.rs +++ b/packages/ciphernode/data/src/into_key.rs @@ -10,7 +10,7 @@ impl IntoKey for Vec { } } -+/// Keys can be references to vectors of bytes (&Vec) +/// Keys can be references to vectors of bytes (&Vec) impl IntoKey for &Vec { fn into_key(self) -> Vec { self.clone() diff --git a/packages/ciphernode/data/src/sled_store.rs b/packages/ciphernode/data/src/sled_store.rs index e87eec9d..b140be9d 100644 --- a/packages/ciphernode/data/src/sled_store.rs +++ b/packages/ciphernode/data/src/sled_store.rs @@ -15,7 +15,8 @@ impl Actor for SledStore { impl SledStore { pub fn new(bus: &Addr, path: &str) -> Result { - let db = sled::open(path).context("could not open db")?; + let db = sled::open(path) + .with_context(|| format!("Could not open database at path '{}'", path))?; Ok(Self { db, bus: bus.clone(), From 683895542463ca9fab66b28cf0be25508fc16443 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 14:43:26 +1100 Subject: [PATCH 44/49] Remove clone --- packages/ciphernode/data/src/snapshot.rs | 2 +- packages/ciphernode/router/src/e3_request_router.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ciphernode/data/src/snapshot.rs b/packages/ciphernode/data/src/snapshot.rs index f747a85a..ba11d9cd 100644 --- a/packages/ciphernode/data/src/snapshot.rs +++ b/packages/ciphernode/data/src/snapshot.rs @@ -22,7 +22,7 @@ pub trait Checkpoint: Snapshot { /// Declare the DataStore instance available on the object fn repository(&self) -> Repository; - /// Write the current snapshot to the DataStore provided by `get_store()` at the object's id returned by `get_id()` + /// Write the current snapshot to the `Repository` provided by `repository()` fn checkpoint(&self) { self.repository().write(&self.snapshot()); } diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index 86684c6f..8f72d9bb 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -132,7 +132,7 @@ impl Handler for E3RequestRouter { }) }); - for feature in self.features.clone().iter() { + for feature in self.features.iter() { feature.on_event(context, &msg); } From fc3518cf4e5035d1dc0b35113e007419500dd8f6 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 14:47:09 +1100 Subject: [PATCH 45/49] Update comments --- packages/ciphernode/router/src/e3_request_router.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/ciphernode/router/src/e3_request_router.rs b/packages/ciphernode/router/src/e3_request_router.rs index 8f72d9bb..943a3691 100644 --- a/packages/ciphernode/router/src/e3_request_router.rs +++ b/packages/ciphernode/router/src/e3_request_router.rs @@ -38,11 +38,8 @@ impl EventBuffer { .unwrap_or_default() } } -/// Format of the hook that needs to be passed to E3RequestRouter -// pub type EventHook = Box; -// pub type Hydrator = Box; -// pub type E3Feature = (EventHook, Hydrator); +/// Format of the hook that needs to be passed to E3RequestRouter #[async_trait] pub trait E3Feature: Send + Sync + 'static { fn on_event(&self, ctx: &mut E3RequestContext, evt: &EnclaveEvent); From e3962e24b44e295fda0358c649ca3736ce717988 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 14:49:34 +1100 Subject: [PATCH 46/49] Update types --- packages/ciphernode/evm/src/event_reader.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/ciphernode/evm/src/event_reader.rs b/packages/ciphernode/evm/src/event_reader.rs index c64dd775..9b225df6 100644 --- a/packages/ciphernode/evm/src/event_reader.rs +++ b/packages/ciphernode/evm/src/event_reader.rs @@ -2,11 +2,9 @@ use crate::helpers::{self, create_readonly_provider, ensure_ws_rpc, ReadonlyProv use actix::prelude::*; use actix::{Addr, Recipient}; use alloy::primitives::{LogData, B256}; -use alloy::{ - eips::BlockNumberOrTag, primitives::Address, rpc::types::Filter, sol, sol_types::SolEvent, -}; +use alloy::{eips::BlockNumberOrTag, primitives::Address, rpc::types::Filter}; use anyhow::{anyhow, Result}; -use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus, Shutdown, Subscribe}; +use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent, EventBus, Subscribe}; use tokio::sync::oneshot; use tracing::info; @@ -17,7 +15,7 @@ pub struct EvmEventReader { provider: Option, contract_address: Address, bus: Recipient, - extractor: fn(&LogData, Option<&B256>, u64) -> Option, + extractor: ExtractorFn, shutdown_rx: Option>, shutdown_tx: Option>, } From 716a067043a37d35a4a047fa9415f0b997ffe542 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 14:55:16 +1100 Subject: [PATCH 47/49] Remove unwrap --- packages/ciphernode/fhe/src/fhe.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ciphernode/fhe/src/fhe.rs b/packages/ciphernode/fhe/src/fhe.rs index 99fd8112..f5131aec 100644 --- a/packages/ciphernode/fhe/src/fhe.rs +++ b/packages/ciphernode/fhe/src/fhe.rs @@ -139,7 +139,7 @@ impl Snapshot for Fhe { impl FromSnapshotWithParams for Fhe { type Params = SharedRng; async fn from_snapshot(rng: SharedRng, snapshot: FheSnapshot) -> Result { - let params = Arc::new(BfvParameters::try_deserialize(&snapshot.params).unwrap()); + let params = Arc::new(BfvParameters::try_deserialize(&snapshot.params)?); let crp = CommonRandomPoly::deserialize(&snapshot.crp, ¶ms)?; Ok(Fhe::new(params, crp, rng)) } From 612424f05e5f2ae49569cee3a875e9e5aaf7fb50 Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 15:00:29 +1100 Subject: [PATCH 48/49] Add comment --- packages/ciphernode/data/src/snapshot.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/ciphernode/data/src/snapshot.rs b/packages/ciphernode/data/src/snapshot.rs index ba11d9cd..50b8a71e 100644 --- a/packages/ciphernode/data/src/snapshot.rs +++ b/packages/ciphernode/data/src/snapshot.rs @@ -38,6 +38,8 @@ pub trait FromSnapshotWithParams: Snapshot { async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result; } + +/// Enable the self type to be reconstituted from the Snapshot only #[async_trait] pub trait FromSnapshot: Snapshot { /// Return an instance of the persistable object at the state given by the snapshot From b968cd8fb90c0e0f93529e99ce4631234f3d9ffa Mon Sep 17 00:00:00 2001 From: ktdlr Date: Tue, 22 Oct 2024 15:07:02 +1100 Subject: [PATCH 49/49] Formatting --- packages/ciphernode/data/src/snapshot.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/ciphernode/data/src/snapshot.rs b/packages/ciphernode/data/src/snapshot.rs index 50b8a71e..814caaaf 100644 --- a/packages/ciphernode/data/src/snapshot.rs +++ b/packages/ciphernode/data/src/snapshot.rs @@ -38,7 +38,6 @@ pub trait FromSnapshotWithParams: Snapshot { async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result; } - /// Enable the self type to be reconstituted from the Snapshot only #[async_trait] pub trait FromSnapshot: Snapshot {