diff --git a/Cargo.lock b/Cargo.lock index 1eda8e84a832e..1ecbc5033573d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4177,9 +4177,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" dependencies = [ "anstream", "anstyle", @@ -6705,9 +6705,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" dependencies = [ "serde", ] @@ -13886,19 +13886,23 @@ dependencies = [ "bcs", "bytes", "clap", + "env_logger", + "log", "move-binary-format", "move-core-types", + "object_store", + "reqwest 0.12.5", "serde", "serde_json", "serde_yaml 0.8.26", "sui-config", - "sui-json", "sui-json-rpc-types", "sui-package-resolver", "sui-rest-api", "sui-sdk 1.34.0", "sui-types", "tokio", + "url", ] [[package]] diff --git a/crates/sui-light-client/.gitignore b/crates/sui-light-client/.gitignore new file mode 100644 index 0000000000000..c2ec96c36f8b7 --- /dev/null +++ b/crates/sui-light-client/.gitignore @@ -0,0 +1 @@ +checkpoints_dir/* \ No newline at end of file diff --git a/crates/sui-light-client/Cargo.toml b/crates/sui-light-client/Cargo.toml index a18d7df4c49cd..7992b77d71bb0 100644 --- a/crates/sui-light-client/Cargo.toml +++ b/crates/sui-light-client/Cargo.toml @@ -27,8 +27,12 @@ serde_json.workspace = true sui-types.workspace = true sui-config.workspace = true sui-rest-api.workspace = true -sui-json.workspace = true sui-sdk.workspace = true move-binary-format.workspace = true sui-json-rpc-types.workspace = true sui-package-resolver.workspace = true +url.workspace = true +reqwest.workspace = true +object_store.workspace = true +env_logger = "0.11.5" +log = "0.4.22" diff --git a/crates/sui-light-client/example_config/light_client.yaml b/crates/sui-light-client/example_config/light_client.yaml index 8d3fc8688af3e..0f45d70bfafc9 100644 --- a/crates/sui-light-client/example_config/light_client.yaml +++ b/crates/sui-light-client/example_config/light_client.yaml @@ -1,3 +1,5 @@ -full_node_url: "http://ord-mnt-rpcbig-06.mainnet.sui.io:9000" +full_node_url: "https://fullnode.mainnet.sui.io:443" checkpoint_summary_dir: "checkpoints_dir" -genesis_filename: "genesis.blob" \ No newline at end of file +genesis_filename: "genesis.blob" +object_store_url: "https://checkpoints.mainnet.sui.io" +graphql_url: "https://sui-mainnet.mystenlabs.com/graphql" \ No newline at end of file diff --git a/crates/sui-light-client/src/main.rs b/crates/sui-light-client/src/main.rs index 0dc3f586f4ee5..f335e5cde7e94 100644 --- a/crates/sui-light-client/src/main.rs +++ b/crates/sui-light-client/src/main.rs @@ -4,9 +4,9 @@ use anyhow::anyhow; use async_trait::async_trait; use move_core_types::account_address::AccountAddress; -use sui_json_rpc_types::SuiTransactionBlockResponseOptions; +use sui_json_rpc_types::{SuiObjectDataOptions, SuiTransactionBlockResponseOptions}; -use sui_rest_api::{CheckpointData, Client}; +use sui_rest_api::CheckpointData; use sui_types::{ base_types::ObjectID, committee::Committee, @@ -15,20 +15,26 @@ use sui_types::{ effects::{TransactionEffects, TransactionEffectsAPI, TransactionEvents}, message_envelope::Envelope, messages_checkpoint::{CertifiedCheckpointSummary, CheckpointSummary, EndOfEpochData}, - object::{Data, Object}, + object::{bounded_visitor::BoundedVisitor, Data, Object}, }; use sui_config::genesis::Genesis; -use sui_json::SuiJsonValue; use sui_package_resolver::Result as ResolverResult; use sui_package_resolver::{Package, PackageStore, Resolver}; use sui_sdk::SuiClientBuilder; use clap::{Parser, Subcommand}; -use std::{fs, io::Write, path::PathBuf, str::FromStr}; +use std::{collections::HashMap, fs, io::Write, path::PathBuf, str::FromStr, sync::Mutex}; use std::{io::Read, sync::Arc}; +use log::info; +use object_store::parse_url; +use object_store::path::Path; +use serde_json::json; +use serde_json::Value; +use url::Url; + /// A light client for the Sui blockchain #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -43,11 +49,15 @@ struct Args { struct RemotePackageStore { config: Config, + cache: Mutex>>, } impl RemotePackageStore { pub fn new(config: Config) -> Self { - Self { config } + Self { + config, + cache: Mutex::new(HashMap::new()), + } } } @@ -56,9 +66,21 @@ impl PackageStore for RemotePackageStore { /// Read package contents. Fails if `id` is not an object, not a package, or is malformed in /// some way. async fn fetch(&self, id: AccountAddress) -> ResolverResult> { + // Check if we have it in the cache + if let Some(package) = self.cache.lock().unwrap().get(&id) { + info!("Fetch Package: {} cache hit", id); + return Ok(package.clone()); + } + + info!("Fetch Package: {}", id); + let object = get_verified_object(&self.config, id.into()).await.unwrap(); - let package = Package::read_from_object(&object).unwrap(); - Ok(Arc::new(package)) + let package = Arc::new(Package::read_from_object(&object).unwrap()); + + // Add to the cache + self.cache.lock().unwrap().insert(id, package.clone()); + + Ok(package) } } @@ -93,12 +115,41 @@ struct Config { // Genesis file name genesis_filename: PathBuf, + + /// Object store url + object_store_url: String, + + /// GraphQL endpoint + graphql_url: String, } -impl Config { - pub fn rest_url(&self) -> &str { - &self.full_node_url - } +async fn query_last_checkpoint_of_epoch(config: &Config, epoch_id: u64) -> anyhow::Result { + // GraphQL query to get the last checkpoint of an epoch + let query = json!({ + "query": "query ($epochID: Int) { epoch(id: $epochID) { checkpoints(last: 1) { nodes { sequenceNumber } } } }", + "variables": { "epochID": epoch_id } + }); + + // Submit the query by POSTing to the GraphQL endpoint + let client = reqwest::Client::new(); + let resp = client + .post(&config.graphql_url) + .header("Content-Type", "application/json") + .body(query.to_string()) + .send() + .await + .expect("Cannot connect to graphql") + .text() + .await + .expect("Cannot parse response"); + + // Parse the JSON response to get the last checkpoint of the epoch + let v: Value = serde_json::from_str(resp.as_str()).expect("Incorrect JSON response"); + let checkpoint_number = v["data"]["epoch"]["checkpoints"]["nodes"][0]["sequenceNumber"] + .as_u64() + .unwrap(); + + Ok(checkpoint_number) } // The list of checkpoints at the end of each epoch @@ -182,11 +233,19 @@ fn write_checkpoint_list( async fn download_checkpoint_summary( config: &Config, - seq: u64, + checkpoint_number: u64, ) -> anyhow::Result { // Download the checkpoint from the server - let client = Client::new(config.rest_url()); - client.get_checkpoint_summary(seq).await.map_err(Into::into) + + let url = Url::parse(&config.object_store_url)?; + let (dyn_store, _store_path) = parse_url(&url).unwrap(); + let path = Path::from(format!("{}.chk", checkpoint_number)); + let response = dyn_store.get(&path).await?; + let bytes = response.bytes().await?; + let (_, blob) = bcs::from_bytes::<(u8, CheckpointData)>(&bytes)?; + + info!("Downloaded checkpoint summary: {}", checkpoint_number); + Ok(blob.checkpoint_summary) } /// Run binary search to for each end of epoch checkpoint that is missing @@ -202,73 +261,58 @@ async fn sync_checkpoint_list_to_latest(config: &Config) -> anyhow::Result<()> { // Download the latest in list checkpoint let summary = download_checkpoint_summary(config, *latest_in_list).await?; let mut last_epoch = summary.epoch(); - let mut last_checkpoint_seq = summary.sequence_number; // Download the very latest checkpoint - let client = Client::new(config.rest_url()); - let latest = client.get_latest_checkpoint().await?; - - // Binary search to find missing checkpoints + let client = SuiClientBuilder::default() + .build(config.full_node_url.as_str()) + .await + .expect("Cannot connect to full node"); + + let latest_seq = client + .read_api() + .get_latest_checkpoint_sequence_number() + .await?; + let latest = download_checkpoint_summary(config, latest_seq).await?; + + // Sequentially record all the missing end of epoch checkpoints numbers while last_epoch + 1 < latest.epoch() { - let mut start = last_checkpoint_seq; - let mut end = latest.sequence_number; - let target_epoch = last_epoch + 1; - // Print target - println!("Target Epoch: {}", target_epoch); - let mut found_summary = None; - - while start < end { - let mid = (start + end) / 2; - let summary = download_checkpoint_summary(config, mid).await?; - - // print summary epoch and seq - println!( - "Epoch: {} Seq: {}: {}", - summary.epoch(), - summary.sequence_number, - summary.end_of_epoch_data.is_some() - ); - - if summary.epoch() == target_epoch && summary.end_of_epoch_data.is_some() { - found_summary = Some(summary); - break; - } + let target_last_checkpoint_number = + query_last_checkpoint_of_epoch(config, target_epoch).await?; - if summary.epoch() <= target_epoch { - start = mid + 1; - } else { - end = mid; - } - } + // Add to the list + checkpoints_list + .checkpoints + .push(target_last_checkpoint_number); + write_checkpoint_list(config, &checkpoints_list)?; - if let Some(summary) = found_summary { - // Note: Do not write summary to file, since we must only persist - // checkpoints that have been verified by the previous committee + // Update + last_epoch = target_epoch; - // Add to the list - checkpoints_list.checkpoints.push(summary.sequence_number); - write_checkpoint_list(config, &checkpoints_list)?; - - // Update - last_epoch = summary.epoch(); - last_checkpoint_seq = summary.sequence_number; - } + println!( + "Last Epoch: {} Last Checkpoint: {}", + target_epoch, target_last_checkpoint_number + ); } Ok(()) } async fn check_and_sync_checkpoints(config: &Config) -> anyhow::Result<()> { - sync_checkpoint_list_to_latest(config).await?; + sync_checkpoint_list_to_latest(config) + .await + .map_err(|e| anyhow!(format!("Cannot refresh list: {e}")))?; // Get the local checkpoint list - let checkpoints_list: CheckpointsList = read_checkpoint_list(config)?; + let checkpoints_list: CheckpointsList = read_checkpoint_list(config) + .map_err(|e| anyhow!(format!("Cannot read checkpoint list: {e}")))?; // Load the genesis committee let mut genesis_path = config.checkpoint_summary_dir.clone(); genesis_path.push(&config.genesis_filename); - let genesis_committee = Genesis::load(&genesis_path)?.committee()?; + let genesis_committee = Genesis::load(&genesis_path)? + .committee() + .map_err(|e| anyhow!(format!("Cannot load Genesis: {e}")))?; // Check the signatures of all checkpoints // And download any missing ones @@ -281,10 +325,13 @@ async fn check_and_sync_checkpoints(config: &Config) -> anyhow::Result<()> { // If file exists read the file otherwise download it from the server let summary = if checkpoint_path.exists() { - read_checkpoint(config, *ckp_id)? + read_checkpoint(config, *ckp_id) + .map_err(|e| anyhow!(format!("Cannot read checkpoint: {e}")))? } else { // Download the checkpoint from the server - let summary = download_checkpoint_summary(config, *ckp_id).await?; + let summary = download_checkpoint_summary(config, *ckp_id) + .await + .map_err(|e| anyhow!(format!("Cannot download summary: {e}")))?; summary.clone().try_into_verified(&prev_committee)?; // Write the checkpoint summary to a file write_checkpoint(config, &summary)?; @@ -317,11 +364,21 @@ async fn check_and_sync_checkpoints(config: &Config) -> anyhow::Result<()> { Ok(()) } -async fn get_full_checkpoint(config: &Config, seq: u64) -> anyhow::Result { - // Downloading the checkpoint from the server - let client: Client = Client::new(config.rest_url()); - let full_checkpoint = client.get_full_checkpoint(seq).await?; - +async fn get_full_checkpoint( + config: &Config, + checkpoint_number: u64, +) -> anyhow::Result { + let url = Url::parse(&config.object_store_url) + .map_err(|_| anyhow!("Cannot parse object store URL"))?; + let (dyn_store, _store_path) = parse_url(&url).unwrap(); + let path = Path::from(format!("{}.chk", checkpoint_number)); + info!("Request full checkpoint: {}", path); + let response = dyn_store + .get(&path) + .await + .map_err(|_| anyhow!("Cannot get full checkpoint from object store"))?; + let bytes = response.bytes().await?; + let (_, full_checkpoint) = bcs::from_bytes::<(u8, CheckpointData)>(&bytes)?; Ok(full_checkpoint) } @@ -363,24 +420,26 @@ async fn get_verified_effects_and_events( config: &Config, tid: TransactionDigest, ) -> anyhow::Result<(TransactionEffects, Option)> { - let sui_mainnet: Arc = Arc::new( - SuiClientBuilder::default() - .build(config.full_node_url.as_str()) - .await - .unwrap(), - ); + let sui_mainnet: sui_sdk::SuiClient = SuiClientBuilder::default() + .build(config.full_node_url.as_str()) + .await + .unwrap(); let read_api = sui_mainnet.read_api(); + info!("Getting effects and events for TID: {}", tid); // Lookup the transaction id and get the checkpoint sequence number let options = SuiTransactionBlockResponseOptions::new(); let seq = read_api .get_transaction_with_options(tid, options) - .await? + .await + .map_err(|e| anyhow!(format!("Cannot get transaction: {e}")))? .checkpoint .ok_or(anyhow!("Transaction not found"))?; // Download the full checkpoint for this sequence number - let full_check_point = get_full_checkpoint(config, seq).await?; + let full_check_point = get_full_checkpoint(config, seq) + .await + .map_err(|e| anyhow!(format!("Cannot get full checkpoint: {e}")))?; // Load the list of stored checkpoints let checkpoints_list: CheckpointsList = read_checkpoint_list(config)?; @@ -420,31 +479,57 @@ async fn get_verified_effects_and_events( // Since we did not find a small committee checkpoint we use the genesis let mut genesis_path = config.checkpoint_summary_dir.clone(); genesis_path.push(&config.genesis_filename); - Genesis::load(&genesis_path)?.committee()? + Genesis::load(&genesis_path)? + .committee() + .map_err(|e| anyhow!(format!("Cannot load Genesis: {e}")))? }; + info!("Extracting effects and events for TID: {}", tid); extract_verified_effects_and_events(&full_check_point, &committee, tid) + .map_err(|e| anyhow!(format!("Cannot extract effects and events: {e}"))) } async fn get_verified_object(config: &Config, id: ObjectID) -> anyhow::Result { - let client: Client = Client::new(config.rest_url()); - let object = client.get_object(id).await?; + let sui_client: Arc = Arc::new( + SuiClientBuilder::default() + .build(config.full_node_url.as_str()) + .await + .unwrap(), + ); + + info!("Getting object: {}", id); + + let read_api = sui_client.read_api(); + let object_json = read_api + .get_object_with_options(id, SuiObjectDataOptions::bcs_lossless()) + .await + .expect("Cannot get object"); + let object = object_json + .into_object() + .expect("Cannot make into object data"); + let object: Object = object.try_into().expect("Cannot reconstruct object"); // Need to authenticate this object - let (effects, _) = get_verified_effects_and_events(config, object.previous_transaction).await?; + let (effects, _) = get_verified_effects_and_events(config, object.previous_transaction) + .await + .expect("Cannot get effects and events"); // check that this object ID, version and hash is in the effects + let target_object_ref = object.compute_object_reference(); effects .all_changed_objects() .iter() - .find(|object_ref| object_ref.0 == object.compute_object_reference()) - .ok_or(anyhow!("Object not found"))?; + .find(|object_ref| object_ref.0 == target_object_ref) + .ok_or(anyhow!("Object not found")) + .expect("Object not found"); Ok(object) } #[tokio::main] pub async fn main() { + env_logger::init(); + // Command line arguments and config loading let args = Args::parse(); @@ -479,23 +564,27 @@ pub async fn main() { exec_digests.transaction, exec_digests.effects ); - for event in events.as_ref().unwrap().data.iter() { - let type_layout = resolver - .type_layout(event.type_.clone().into()) - .await - .unwrap(); - - let json_val = - SuiJsonValue::from_bcs_bytes(Some(&type_layout), &event.contents).unwrap(); + if events.is_some() { + for event in events.as_ref().unwrap().data.iter() { + let type_layout = resolver + .type_layout(event.type_.clone().into()) + .await + .unwrap(); - println!( - "Event:\n - Package: {}\n - Module: {}\n - Sender: {}\n - Type: {}\n{}", - event.package_id, - event.transaction_module, - event.sender, - event.type_, - serde_json::to_string_pretty(&json_val.to_json_value()).unwrap() - ); + let result = BoundedVisitor::deserialize_value(&event.contents, &type_layout) + .expect("Cannot deserialize"); + + println!( + "Event:\n - Package: {}\n - Module: {}\n - Sender: {}\n - Type: {}\n{}", + event.package_id, + event.transaction_module, + event.sender, + event.type_, + serde_json::to_string_pretty(&result).unwrap() + ); + } + } else { + println!("No events found"); } } Some(SCommands::Object { oid }) => { @@ -510,9 +599,9 @@ pub async fn main() { .await .unwrap(); - let json_val = - SuiJsonValue::from_bcs_bytes(Some(&type_layout), move_object.contents()) - .unwrap(); + let result = + BoundedVisitor::deserialize_value(move_object.contents(), &type_layout) + .expect("Cannot deserialize"); let (oid, version, hash) = object.compute_object_reference(); println!( @@ -522,7 +611,7 @@ pub async fn main() { hash, object.owner, object_type, - serde_json::to_string_pretty(&json_val.to_json_value()).unwrap() + serde_json::to_string_pretty(&result).unwrap() ); } } diff --git a/crates/sui-light-client/src/proof.rs b/crates/sui-light-client/src/proof.rs index 8d846384e64bb..e337b9ffa346d 100644 --- a/crates/sui-light-client/src/proof.rs +++ b/crates/sui-light-client/src/proof.rs @@ -3,6 +3,7 @@ use anyhow::anyhow; +use serde::{Deserialize, Serialize}; use sui_types::{ base_types::ObjectRef, committee::Committee, @@ -13,8 +14,8 @@ use sui_types::{ transaction::Transaction, }; -/// Define aspect of Sui state that needs to be certified in a proof -#[derive(Default)] +/// Define aspect of Sui state that need to be certified in a proof +#[derive(Default, Debug, Serialize, Deserialize)] pub struct ProofTarget { /// Objects that need to be certified. pub objects: Vec<(ObjectRef, Object)>, @@ -58,6 +59,7 @@ impl ProofTarget { /// Part of a proof that provides evidence relating to a specific transaction to /// certify objects and events. +#[derive(Debug, Serialize, Deserialize)] pub struct TransactionProof { /// Checkpoint contents including this transaction. pub checkpoint_contents: CheckpointContents, @@ -74,6 +76,7 @@ pub struct TransactionProof { /// A proof for specific targets. It certifies a checkpoint summary and optionally includes /// transaction evidence to certify objects and events. +#[derive(Debug, Serialize, Deserialize)] pub struct Proof { /// Targets of the proof are a committee, objects, or events that need to be certified. pub targets: ProofTarget,