From fcf70166c98c374d9732146d1fb10d859f066496 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 11 Nov 2022 13:16:06 -0500 Subject: [PATCH 01/12] feat: improve verbosity in chain observer --- .../src/indexer/bitcoin/blocks_pool.rs | 7 +- .../src/indexer/bitcoin/mod.rs | 20 ++++-- .../src/indexer/mod.rs | 11 +-- .../src/indexer/stacks/blocks_pool.rs | 7 +- .../src/observer/mod.rs | 70 ++++++++++++------- 5 files changed, 75 insertions(+), 40 deletions(-) diff --git a/components/chainhook-event-observer/src/indexer/bitcoin/blocks_pool.rs b/components/chainhook-event-observer/src/indexer/bitcoin/blocks_pool.rs index ee04e8adf..3e7c791f9 100644 --- a/components/chainhook-event-observer/src/indexer/bitcoin/blocks_pool.rs +++ b/components/chainhook-event-observer/src/indexer/bitcoin/blocks_pool.rs @@ -30,7 +30,7 @@ impl BitcoinBlockPool { pub fn process_block( &mut self, block: BitcoinBlockData, - ) -> Result, ()> { + ) -> Result, String> { info!("Start processing Bitcoin {}", block.block_identifier); // Keep block data in memory @@ -52,7 +52,10 @@ impl BitcoinBlockPool { let previous_canonical_fork_id = self.canonical_fork_id; let previous_canonical_fork = match self.forks.get(&previous_canonical_fork_id) { Some(fork) => fork.clone(), - None => return Err(()), + None => { + error!("unable to retrieve previous bitcoin fork"); + return Ok(None); + } }; let mut fork_updated = None; diff --git a/components/chainhook-event-observer/src/indexer/bitcoin/mod.rs b/components/chainhook-event-observer/src/indexer/bitcoin/mod.rs index e258dcbbf..d51c52470 100644 --- a/components/chainhook-event-observer/src/indexer/bitcoin/mod.rs +++ b/components/chainhook-event-observer/src/indexer/bitcoin/mod.rs @@ -37,24 +37,30 @@ pub struct RewardParticipant { pub fn standardize_bitcoin_block( indexer_config: &IndexerConfig, marshalled_block: JsonValue, -) -> BitcoinBlockData { +) -> Result { let auth = Auth::UserPass( indexer_config.bitcoin_node_rpc_username.clone(), indexer_config.bitcoin_node_rpc_password.clone(), ); - - let rpc = Client::new(&indexer_config.bitcoin_node_rpc_url, auth).unwrap(); - - let partial_block: NewBitcoinBlock = serde_json::from_value(marshalled_block).unwrap(); + let rpc = Client::new(&indexer_config.bitcoin_node_rpc_url, auth).map_err(|e| { + format!( + "unable for bitcoin rpc initialize client: {}", + e.to_string() + ) + })?; + let partial_block: NewBitcoinBlock = serde_json::from_value(marshalled_block) + .map_err(|e| format!("unable for parse bitcoin block: {}", e.to_string()))?; let block_hash = { let block_hash_str = partial_block.burn_block_hash.strip_prefix("0x").unwrap(); let mut block_hash_bytes = hex_bytes(&block_hash_str).unwrap(); block_hash_bytes.reverse(); BlockHash::from_slice(&block_hash_bytes).unwrap() }; - let block = rpc.get_block(&block_hash).unwrap(); + let block = rpc + .get_block(&block_hash) + .map_err(|e| format!("unable for invoke rpc get_block: {}", e.to_string()))?; let block_height = partial_block.burn_block_height; - build_block(block, block_height, indexer_config) + Ok(build_block(block, block_height, indexer_config)) } pub fn build_block( diff --git a/components/chainhook-event-observer/src/indexer/mod.rs b/components/chainhook-event-observer/src/indexer/mod.rs index 61aeccb08..0a1372d69 100644 --- a/components/chainhook-event-observer/src/indexer/mod.rs +++ b/components/chainhook-event-observer/src/indexer/mod.rs @@ -67,15 +67,16 @@ impl Indexer { pub fn handle_bitcoin_block( &mut self, marshalled_block: JsonValue, - ) -> Result, ()> { - let block = bitcoin::standardize_bitcoin_block(&self.config, marshalled_block); - self.bitcoin_blocks_pool.process_block(block) + ) -> Result, String> { + let block = bitcoin::standardize_bitcoin_block(&self.config, marshalled_block)?; + let event = self.bitcoin_blocks_pool.process_block(block); + event } pub fn handle_stacks_serialized_block( &mut self, serialized_block: &str, - ) -> Result, ()> { + ) -> Result, String> { let block = stacks::standardize_stacks_serialized_block( &self.config, serialized_block, @@ -87,7 +88,7 @@ impl Indexer { pub fn handle_stacks_marshalled_block( &mut self, marshalled_block: JsonValue, - ) -> Result, ()> { + ) -> Result, String> { let block = stacks::standardize_stacks_marshalled_block( &self.config, marshalled_block, diff --git a/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs b/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs index f2bae02a5..aa70373c4 100644 --- a/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs +++ b/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs @@ -40,7 +40,7 @@ impl StacksBlockPool { pub fn process_block( &mut self, block: StacksBlockData, - ) -> Result, ()> { + ) -> Result, String> { info!("Start processing Stacks {}", block.block_identifier); // Keep block data in memory @@ -62,7 +62,10 @@ impl StacksBlockPool { let previous_canonical_fork_id = self.canonical_fork_id; let previous_canonical_fork = match self.forks.get(&previous_canonical_fork_id) { Some(fork) => fork.clone(), - None => return Err(()), + None => { + error!("unable to retrieve previous stacks fork"); + return Ok(None); + } }; let mut fork_updated = None; diff --git a/components/chainhook-event-observer/src/observer/mod.rs b/components/chainhook-event-observer/src/observer/mod.rs index ccdd25fcb..f3a975272 100644 --- a/components/chainhook-event-observer/src/observer/mod.rs +++ b/components/chainhook-event-observer/src/observer/mod.rs @@ -836,22 +836,36 @@ pub fn handle_new_bitcoin_block( // into account the last 7 blocks. let chain_update = match indexer_rw_lock.inner().write() { Ok(mut indexer) => indexer.handle_bitcoin_block(marshalled_block.into_inner()), - _ => { + Err(e) => { + warn!("unable to acquire indexer_rw_lock: {}", e.to_string()); return Json(json!({ "status": 500, "result": "Unable to acquire lock", - })) + })); } }; - if let Ok(Some(chain_event)) = chain_update { - let background_job_tx = background_job_tx.inner(); - match background_job_tx.lock() { - Ok(tx) => { - let _ = tx.send(ObserverCommand::PropagateBitcoinChainEvent(chain_event)); - } - _ => {} - }; + match chain_update { + Ok(Some(chain_event)) => { + match background_job_tx.lock() { + Ok(tx) => { + let _ = tx.send(ObserverCommand::PropagateBitcoinChainEvent(chain_event)); + } + Err(e) => { + warn!("unable to acquire background_job_tx: {}", e.to_string()); + return Json(json!({ + "status": 500, + "result": "Unable to acquire lock", + })); + } + }; + } + Ok(None) => { + info!("unable to infer chain progress"); + } + Err(e) => { + error!("unable to handle bitcoin block: {}", e) + } } Json(json!({ @@ -887,20 +901,28 @@ pub fn handle_new_stacks_block( } }; - if let Ok(Some(chain_event)) = chain_event { - let background_job_tx = background_job_tx.inner(); - match background_job_tx.lock() { - Ok(tx) => { - let _ = tx.send(ObserverCommand::PropagateStacksChainEvent(chain_event)); - } - Err(e) => { - warn!("unable to acquire background_job_tx: {}", e.to_string()); - return Json(json!({ - "status": 500, - "result": "Unable to acquire lock", - })); - } - }; + match chain_event { + Ok(Some(chain_event)) => { + let background_job_tx = background_job_tx.inner(); + match background_job_tx.lock() { + Ok(tx) => { + let _ = tx.send(ObserverCommand::PropagateStacksChainEvent(chain_event)); + } + Err(e) => { + warn!("unable to acquire background_job_tx: {}", e.to_string()); + return Json(json!({ + "status": 500, + "result": "Unable to acquire lock", + })); + } + }; + } + Ok(None) => { + info!("unable to infer chain progress"); + } + Err(e) => { + error!("unable to handle stacks block: {}", e) + } } Json(json!({ From 353ceb617b8a5b710331fe3387b07f6ad48e3f48 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 11 Nov 2022 14:10:21 -0500 Subject: [PATCH 02/12] chore: better error management in event observer --- .../src/indexer/mod.rs | 12 +- .../src/indexer/stacks/blocks_pool.rs | 11 +- .../src/indexer/stacks/mod.rs | 221 +++++++++--------- .../src/indexer/tests/mod.rs | 4 +- .../src/observer/mod.rs | 37 ++- .../chainhook-node/src/block/digestion.rs | 10 +- .../chainhook-node/src/block/ingestion.rs | 10 +- 7 files changed, 166 insertions(+), 139 deletions(-) diff --git a/components/chainhook-event-observer/src/indexer/mod.rs b/components/chainhook-event-observer/src/indexer/mod.rs index 0a1372d69..e6c5d1b7c 100644 --- a/components/chainhook-event-observer/src/indexer/mod.rs +++ b/components/chainhook-event-observer/src/indexer/mod.rs @@ -81,7 +81,7 @@ impl Indexer { &self.config, serialized_block, &mut self.stacks_context, - ); + )?; self.stacks_blocks_pool.process_block(block) } @@ -93,31 +93,31 @@ impl Indexer { &self.config, marshalled_block, &mut self.stacks_context, - ); + )?; self.stacks_blocks_pool.process_block(block) } pub fn handle_stacks_serialized_microblock_trail( &mut self, serialized_microblock_trail: &str, - ) -> Option { + ) -> Result, String> { let microblocks = stacks::standardize_stacks_serialized_microblock_trail( &self.config, serialized_microblock_trail, &mut self.stacks_context, - ); + )?; self.stacks_blocks_pool.process_microblocks(microblocks) } pub fn handle_stacks_marshalled_microblock_trail( &mut self, marshalled_microblock_trail: JsonValue, - ) -> Option { + ) -> Result, String> { let microblocks = stacks::standardize_stacks_marshalled_microblock_trail( &self.config, marshalled_microblock_trail, &mut self.stacks_context, - ); + )?; self.stacks_blocks_pool.process_microblocks(microblocks) } diff --git a/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs b/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs index aa70373c4..62c7764e3 100644 --- a/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs +++ b/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs @@ -350,7 +350,7 @@ impl StacksBlockPool { pub fn process_microblocks( &mut self, microblocks: Vec, - ) -> Option { + ) -> Result, String> { info!("Start processing {} microblocks", microblocks.len()); let mut previous_canonical_micro_fork = None; @@ -501,7 +501,7 @@ impl StacksBlockPool { if micro_forks_updated.is_empty() { info!("Unable to process microblocks - inboxed for later"); - return None; + return Ok(None); } else { info!("Microblocks successfully appended"); } @@ -509,9 +509,8 @@ impl StacksBlockPool { let anchor_block_updated = match anchor_block_updated { Some(anchor_block_updated) => anchor_block_updated, None => { - // Microblock was received before its anchorblock - - return None; + info!("Microblock was received before its anchorblock"); + return Ok(None); } }; @@ -552,7 +551,7 @@ impl StacksBlockPool { &previous_canonical_micro_fork, ); - chain_event + Ok(chain_event) } // We got the confirmed canonical microblock trail, diff --git a/components/chainhook-event-observer/src/indexer/stacks/mod.rs b/components/chainhook-event-observer/src/indexer/stacks/mod.rs index e0ec00b77..099c64893 100644 --- a/components/chainhook-event-observer/src/indexer/stacks/mod.rs +++ b/components/chainhook-event-observer/src/indexer/stacks/mod.rs @@ -67,7 +67,7 @@ pub struct NewMicroblockTrail { pub events: Vec, } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] pub struct NewTransaction { pub txid: String, pub tx_index: usize, @@ -115,69 +115,69 @@ pub struct NewEvent { } impl NewEvent { - pub fn into_chainhook_event(&self) -> StacksTransactionEvent { + pub fn into_chainhook_event(&self) -> Result { if let Some(ref event_data) = self.stx_mint_event { let data: STXMintEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::STXMintEvent(data.clone()); + return Ok(StacksTransactionEvent::STXMintEvent(data.clone())); } else if let Some(ref event_data) = self.stx_lock_event { let data: STXLockEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::STXLockEvent(data.clone()); + return Ok(StacksTransactionEvent::STXLockEvent(data.clone())); } else if let Some(ref event_data) = self.stx_burn_event { let data: STXBurnEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::STXBurnEvent(data.clone()); + return Ok(StacksTransactionEvent::STXBurnEvent(data.clone())); } else if let Some(ref event_data) = self.stx_transfer_event { let data: STXTransferEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::STXTransferEvent(data.clone()); + return Ok(StacksTransactionEvent::STXTransferEvent(data.clone())); } else if let Some(ref event_data) = self.nft_mint_event { let data: NFTMintEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::NFTMintEvent(data.clone()); + return Ok(StacksTransactionEvent::NFTMintEvent(data.clone())); } else if let Some(ref event_data) = self.nft_burn_event { let data: NFTBurnEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::NFTBurnEvent(data.clone()); + return Ok(StacksTransactionEvent::NFTBurnEvent(data.clone())); } else if let Some(ref event_data) = self.nft_transfer_event { let data: NFTTransferEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::NFTTransferEvent(data.clone()); + return Ok(StacksTransactionEvent::NFTTransferEvent(data.clone())); } else if let Some(ref event_data) = self.ft_mint_event { let data: FTMintEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::FTMintEvent(data.clone()); + return Ok(StacksTransactionEvent::FTMintEvent(data.clone())); } else if let Some(ref event_data) = self.ft_burn_event { let data: FTBurnEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::FTBurnEvent(data.clone()); + return Ok(StacksTransactionEvent::FTBurnEvent(data.clone())); } else if let Some(ref event_data) = self.ft_transfer_event { let data: FTTransferEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::FTTransferEvent(data.clone()); + return Ok(StacksTransactionEvent::FTTransferEvent(data.clone())); } else if let Some(ref event_data) = self.data_var_set_event { let data: DataVarSetEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::DataVarSetEvent(data.clone()); + return Ok(StacksTransactionEvent::DataVarSetEvent(data.clone())); } else if let Some(ref event_data) = self.data_map_insert_event { let data: DataMapInsertEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::DataMapInsertEvent(data.clone()); + return Ok(StacksTransactionEvent::DataMapInsertEvent(data.clone())); } else if let Some(ref event_data) = self.data_map_update_event { let data: DataMapUpdateEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::DataMapUpdateEvent(data.clone()); + return Ok(StacksTransactionEvent::DataMapUpdateEvent(data.clone())); } else if let Some(ref event_data) = self.data_map_delete_event { let data: DataMapDeleteEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::DataMapDeleteEvent(data.clone()); + return Ok(StacksTransactionEvent::DataMapDeleteEvent(data.clone())); } else if let Some(ref event_data) = self.contract_event { let data: SmartContractEventData = serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); - return StacksTransactionEvent::SmartContractEvent(data.clone()); + return Ok(StacksTransactionEvent::SmartContractEvent(data.clone())); } - unreachable!() + return Err(format!("unable to support event type")); } } @@ -197,25 +197,35 @@ pub struct ContractReadonlyCall { pub fn standardize_stacks_serialized_block_header( serialized_block: &str, -) -> (BlockIdentifier, BlockIdentifier) { - let mut block_header: NewBlockHeader = serde_json::from_str(serialized_block).unwrap(); +) -> Result<(BlockIdentifier, BlockIdentifier), String> { + let mut block_header: NewBlockHeader = serde_json::from_str(serialized_block) + .map_err(|e| format!("unable to parse stacks block_header {}", e.to_string()))?; + let hash = block_header + .index_block_hash + .take() + .ok_or(format!("unable to retrieve index_block_hash"))?; let block_identifier = BlockIdentifier { - hash: block_header.index_block_hash.take().unwrap(), + hash, index: block_header.block_height, }; + let parent_hash = block_header + .parent_index_block_hash + .take() + .ok_or(format!("unable to retrieve parent_index_block_hash"))?; let parent_block_identifier = BlockIdentifier { - hash: block_header.parent_index_block_hash.take().unwrap(), + hash: parent_hash, index: block_identifier.index - 1, }; - (block_identifier, parent_block_identifier) + Ok((block_identifier, parent_block_identifier)) } pub fn standardize_stacks_serialized_block( indexer_config: &IndexerConfig, serialized_block: &str, ctx: &mut StacksChainContext, -) -> StacksBlockData { - let mut block: NewBlock = serde_json::from_str(serialized_block).unwrap(); +) -> Result { + let mut block: NewBlock = serde_json::from_str(serialized_block) + .map_err(|e| format!("unable to parse stacks block_header {}", e.to_string()))?; standardize_stacks_block(indexer_config, &mut block, ctx) } @@ -223,8 +233,9 @@ pub fn standardize_stacks_marshalled_block( indexer_config: &IndexerConfig, marshalled_block: JsonValue, ctx: &mut StacksChainContext, -) -> StacksBlockData { - let mut block: NewBlock = serde_json::from_value(marshalled_block).unwrap(); +) -> Result { + let mut block: NewBlock = serde_json::from_value(marshalled_block) + .map_err(|e| format!("unable to parse stacks block {}", e.to_string()))?; standardize_stacks_block(indexer_config, &mut block, ctx) } @@ -232,7 +243,7 @@ pub fn standardize_stacks_block( indexer_config: &IndexerConfig, block: &mut NewBlock, ctx: &mut StacksChainContext, -) -> StacksBlockData { +) -> Result { let pox_cycle_length: u64 = (ctx.pox_info.prepare_phase_block_length + ctx.pox_info.reward_phase_block_length).into(); let current_len = block.burn_block_height - ctx.pox_info.first_burnchain_block_height; @@ -246,68 +257,49 @@ pub fn standardize_stacks_block( .or_insert(vec![&event]); } - let transactions = block - .transactions - .iter() - .map(|tx| { - let tx_events = events.remove(&tx.txid).unwrap_or(vec![]); - let (description, tx_type, fee, sender, sponsor) = - match get_tx_description(&tx.raw_tx, &tx_events) { - Ok(desc) => desc, - Err(_) => { - return StacksTransactionData { - transaction_identifier: TransactionIdentifier { - hash: "0x00".into(), - }, - operations: vec![], - metadata: StacksTransactionMetadata { - success: true, - result: "0x00".into(), - raw_tx: "0x00".into(), - sender: "0x00".into(), - fee: 0, - sponsor: None, - kind: StacksTransactionKind::Other, - execution_cost: None, - receipt: StacksTransactionReceipt::default(), - description: "Unparsable transaction".into(), - position: StacksTransactionPosition::Index(0), - proof: None, - }, - } - } - }; - let events = tx_events.iter().map(|e| e.into_chainhook_event()).collect(); - let (receipt, operations) = get_standardized_stacks_receipt( - &tx.txid, - events, - &mut ctx.asset_class_map, - &indexer_config.stacks_node_rpc_url, - true, - ); + let mut transactions = vec![]; + for tx in block.transactions.iter() { + let tx_events = events.remove(&tx.txid).unwrap_or(vec![]); + let (description, tx_type, fee, sender, sponsor) = + match get_tx_description(&tx.raw_tx, &tx_events) { + Ok(desc) => desc, + Err(_) => { + return Err(format!("unable to standardize block ({:?})", tx)); + } + }; + let events = tx_events + .iter() + .map(|e| e.into_chainhook_event()) + .collect::, String>>()?; + let (receipt, operations) = get_standardized_stacks_receipt( + &tx.txid, + events, + &mut ctx.asset_class_map, + &indexer_config.stacks_node_rpc_url, + true, + ); - StacksTransactionData { - transaction_identifier: TransactionIdentifier { - hash: tx.txid.clone(), - }, - operations, - metadata: StacksTransactionMetadata { - success: tx.status == "success", - result: get_value_description(&tx.raw_result), - raw_tx: tx.raw_tx.clone(), - sender, - fee, - sponsor, - kind: tx_type, - execution_cost: tx.execution_cost.clone(), - receipt, - description, - position: StacksTransactionPosition::Index(tx.tx_index), - proof: None, - }, - } - }) - .collect(); + transactions.push(StacksTransactionData { + transaction_identifier: TransactionIdentifier { + hash: tx.txid.clone(), + }, + operations, + metadata: StacksTransactionMetadata { + success: tx.status == "success", + result: get_value_description(&tx.raw_result), + raw_tx: tx.raw_tx.clone(), + sender, + fee, + sponsor, + kind: tx_type, + execution_cost: tx.execution_cost.clone(), + receipt, + description, + position: StacksTransactionPosition::Index(tx.tx_index), + proof: None, + }, + }); + } let confirm_microblock_identifier = if block.parent_microblock == "0x0000000000000000000000000000000000000000000000000000000000000000" @@ -323,7 +315,7 @@ pub fn standardize_stacks_block( }) }; - StacksBlockData { + let block = StacksBlockData { block_identifier: BlockIdentifier { hash: block.index_block_hash.clone(), index: block.block_height, @@ -344,16 +336,18 @@ pub fn standardize_stacks_block( confirm_microblock_identifier, }, transactions, - } + }; + Ok(block) } pub fn standardize_stacks_serialized_microblock_trail( indexer_config: &IndexerConfig, serialized_microblock_trail: &str, ctx: &mut StacksChainContext, -) -> Vec { +) -> Result, String> { let mut microblock_trail: NewMicroblockTrail = - serde_json::from_str(serialized_microblock_trail).unwrap(); + serde_json::from_str(serialized_microblock_trail) + .map_err(|e| format!("unable to parse microblock trail {}", e.to_string()))?; standardize_stacks_microblock_trail(indexer_config, &mut microblock_trail, ctx) } @@ -361,9 +355,10 @@ pub fn standardize_stacks_marshalled_microblock_trail( indexer_config: &IndexerConfig, marshalled_microblock_trail: JsonValue, ctx: &mut StacksChainContext, -) -> Vec { +) -> Result, String> { let mut microblock_trail: NewMicroblockTrail = - serde_json::from_value(marshalled_microblock_trail).unwrap(); + serde_json::from_value(marshalled_microblock_trail) + .map_err(|e| format!("unable to parse microblock trail {}", e.to_string()))?; standardize_stacks_microblock_trail(indexer_config, &mut microblock_trail, ctx) } @@ -371,7 +366,7 @@ pub fn standardize_stacks_microblock_trail( indexer_config: &IndexerConfig, microblock_trail: &mut NewMicroblockTrail, ctx: &mut StacksChainContext, -) -> Vec { +) -> Result, String> { let mut events: HashMap<&String, Vec<&NewEvent>> = HashMap::new(); for event in microblock_trail.events.iter() { events @@ -388,7 +383,10 @@ pub fn standardize_stacks_microblock_trail( let (description, tx_type, fee, sender, sponsor) = get_tx_description(&tx.raw_tx, &tx_events).expect("unable to parse transaction"); - let events = tx_events.iter().map(|e| e.into_chainhook_event()).collect(); + let events = tx_events + .iter() + .map(|e| e.into_chainhook_event()) + .collect::, String>>()?; let (receipt, operations) = get_standardized_stacks_receipt( &tx.txid, events, @@ -458,7 +456,7 @@ pub fn standardize_stacks_microblock_trail( } microblocks.sort_by(|a, b| a.block_identifier.cmp(&b.block_identifier)); - microblocks + Ok(microblocks) } pub fn get_value_description(raw_value: &str) -> String { @@ -492,15 +490,15 @@ pub fn get_tx_description( String, // Sender's address Option, // Sponsor's address (optional) ), - (), + String, > { let raw_tx = match raw_tx.strip_prefix("0x") { Some(raw_tx) => raw_tx, - _ => return Err(()), + _ => return Err("unable to read txid".into()), }; let tx_bytes = match hex_bytes(&raw_tx) { Ok(bytes) => bytes, - _ => return Err(()), + Err(e) => return Err(format!("unable to read txid {}", e.to_string())), }; // Handle Stacks transitions operated through Bitcoin transactions @@ -508,13 +506,14 @@ pub fn get_tx_description( let event = match tx_events.first() { Some(event) => event, None => { - debug!("Received block with transaction '0x00' and no events"); - return Err(()); + return Err(format!( + "received block with transaction '0x00' and no events" + )); } }; if let Some(ref event_data) = event.stx_transfer_event { - let data: STXTransferEventData = - serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); + let data: STXTransferEventData = serde_json::from_value(event_data.clone()) + .map_err(|e| format!("unable to decode event_data {}", e.to_string()))?; let description = format!( "transfered: {} µSTX from {} to {} through Bitcoin transaction", data.amount, data.sender, data.recipient @@ -522,8 +521,8 @@ pub fn get_tx_description( let tx_type = StacksTransactionKind::NativeTokenTransfer; return Ok((description, tx_type, 0, data.sender, None)); } else if let Some(ref event_data) = event.stx_lock_event { - let data: STXLockEventData = - serde_json::from_value(event_data.clone()).expect("Unable to decode event_data"); + let data: STXLockEventData = serde_json::from_value(event_data.clone()) + .map_err(|e| format!("unable to decode event_data {}", e.to_string()))?; let description = format!( "stacked: {} µSTX by {} through Bitcoin transaction", data.locked_amount, data.locked_address, @@ -531,13 +530,11 @@ pub fn get_tx_description( let tx_type = StacksTransactionKind::Other; return Ok((description, tx_type, 0, data.locked_address, None)); } - unreachable!() + return Err(format!("unable to parse transaction {raw_tx}")); } - let tx = match StacksTransaction::consensus_deserialize(&mut Cursor::new(&tx_bytes)) { - Ok(bytes) => bytes, - _ => return Err(()), - }; + let tx = StacksTransaction::consensus_deserialize(&mut Cursor::new(&tx_bytes)) + .map_err(|e| format!("unable to consensus decode transaction {}", e.to_string()))?; let (fee, sender, sponsor) = match tx.auth { TransactionAuth::Standard(ref conditions) => ( diff --git a/components/chainhook-event-observer/src/indexer/tests/mod.rs b/components/chainhook-event-observer/src/indexer/tests/mod.rs index d14ab7263..70e5d3c6c 100644 --- a/components/chainhook-event-observer/src/indexer/tests/mod.rs +++ b/components/chainhook-event-observer/src/indexer/tests/mod.rs @@ -16,7 +16,9 @@ pub fn process_stacks_blocks_and_check_expectations( check_chain_event_expectations(chain_event); } BlockEvent::Microblock(microblock) => { - let chain_event = blocks_processor.process_microblocks(vec![microblock]); + let chain_event = blocks_processor + .process_microblocks(vec![microblock]) + .unwrap(); check_chain_event_expectations(chain_event); } } diff --git a/components/chainhook-event-observer/src/observer/mod.rs b/components/chainhook-event-observer/src/observer/mod.rs index f3a975272..fad45c85b 100644 --- a/components/chainhook-event-observer/src/observer/mod.rs +++ b/components/chainhook-event-observer/src/observer/mod.rs @@ -945,28 +945,43 @@ pub fn handle_new_microblocks( info!("POST /new_microblocks"); // Standardize the structure of the microblock, and identify the // kind of update that this new microblock would imply - let mut chain_event = match indexer_rw_lock.inner().write() { + let chain_event = match indexer_rw_lock.inner().write() { Ok(mut indexer) => { let chain_event = indexer .handle_stacks_marshalled_microblock_trail(marshalled_microblock.into_inner()); chain_event } - _ => { + Err(e) => { + warn!("unable to acquire background_job_tx: {}", e.to_string()); return Json(json!({ "status": 500, "result": "Unable to acquire lock", - })) + })); } }; - if let Some(chain_event) = chain_event.take() { - let background_job_tx = background_job_tx.inner(); - match background_job_tx.lock() { - Ok(tx) => { - let _ = tx.send(ObserverCommand::PropagateStacksChainEvent(chain_event)); - } - _ => {} - }; + match chain_event { + Ok(Some(chain_event)) => { + let background_job_tx = background_job_tx.inner(); + match background_job_tx.lock() { + Ok(tx) => { + let _ = tx.send(ObserverCommand::PropagateStacksChainEvent(chain_event)); + } + Err(e) => { + warn!("unable to acquire background_job_tx: {}", e.to_string()); + return Json(json!({ + "status": 500, + "result": "Unable to acquire lock", + })); + } + }; + } + Ok(None) => { + info!("unable to infer chain progress"); + } + Err(e) => { + error!("unable to handle stacks microblock: {}", e); + } } Json(json!({ diff --git a/components/chainhook-node/src/block/digestion.rs b/components/chainhook-node/src/block/digestion.rs index 4a67b8005..2d070691b 100644 --- a/components/chainhook-node/src/block/digestion.rs +++ b/components/chainhook-node/src/block/digestion.rs @@ -39,11 +39,17 @@ pub fn start(command_rx: Receiver, config: &Config) -> Result< let payload: String = con .hget(&key, "blob") .expect("unable to retrieve tip height"); - let block_data = indexer::stacks::standardize_stacks_serialized_block( + let block_data = match indexer::stacks::standardize_stacks_serialized_block( &indexer.config, &payload, &mut indexer.stacks_context, - ); + ) { + Ok(block) => block, + Err(e) => { + error!("unable to handle stacks block: {e}"); + continue; + } + }; let _: Result<(), redis::RedisError> = con.hset_multiple( &key, &[ diff --git a/components/chainhook-node/src/block/ingestion.rs b/components/chainhook-node/src/block/ingestion.rs index a6c99d150..8c35eface 100644 --- a/components/chainhook-node/src/block/ingestion.rs +++ b/components/chainhook-node/src/block/ingestion.rs @@ -103,7 +103,15 @@ pub fn start( while let Ok(Some(record)) = stacks_record_rx.recv() { let (block_identifier, parent_block_identifier) = match &record.kind { RecordKind::StacksBlockReceived => { - indexer::stacks::standardize_stacks_serialized_block_header(&record.raw_log) + match indexer::stacks::standardize_stacks_serialized_block_header( + &record.raw_log, + ) { + Ok(data) => data, + Err(e) => { + error!("unable to handle stacks block: {e}"); + continue; + } + } } _ => unreachable!(), }; From 0f0610229a61d3359e8593d7a40b9bcd4cd24edb Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 11 Nov 2022 16:20:58 -0500 Subject: [PATCH 03/12] refactoring: bitcoin_rpc_url / stacks_rpc_url handling --- .../chainhook-event-observer/src/main.rs | 12 +++---- .../src/observer/mod.rs | 32 +++++-------------- .../src/observer/tests/mod.rs | 6 ++-- components/chainhook-node/src/cli/mod.rs | 12 +++---- components/chainhook-node/src/config/mod.rs | 4 +-- .../stacks-network/src/chains_coordinator.rs | 9 +++--- 6 files changed, 25 insertions(+), 50 deletions(-) diff --git a/components/chainhook-event-observer/src/main.rs b/components/chainhook-event-observer/src/main.rs index f6dfa958b..fb4b78b70 100644 --- a/components/chainhook-event-observer/src/main.rs +++ b/components/chainhook-event-observer/src/main.rs @@ -80,10 +80,8 @@ pub struct EventObserverConfigFile { pub control_port: Option, pub bitcoin_node_username: String, pub bitcoin_node_password: String, - pub bitcoin_node_rpc_host: String, - pub bitcoin_node_rpc_port: u16, - pub stacks_node_rpc_host: String, - pub stacks_node_rpc_port: u16, + pub bitcoin_node_rpc_url: String, + pub stacks_node_rpc_url: String, pub operators: Option>, } @@ -141,10 +139,8 @@ impl EventObserverConfig { .unwrap_or(observer::DEFAULT_CONTROL_PORT), bitcoin_node_username: config_file.bitcoin_node_username.clone(), bitcoin_node_password: config_file.bitcoin_node_password.clone(), - bitcoin_node_rpc_host: config_file.bitcoin_node_rpc_host.clone(), - bitcoin_node_rpc_port: config_file.bitcoin_node_rpc_port.clone(), - stacks_node_rpc_host: config_file.stacks_node_rpc_host.clone(), - stacks_node_rpc_port: config_file.stacks_node_rpc_port.clone(), + bitcoin_node_rpc_url: config_file.bitcoin_node_rpc_url.clone(), + stacks_node_rpc_url: config_file.stacks_node_rpc_url.clone(), operators, display_logs: true, }; diff --git a/components/chainhook-event-observer/src/observer/mod.rs b/components/chainhook-event-observer/src/observer/mod.rs index fad45c85b..be4b3d13b 100644 --- a/components/chainhook-event-observer/src/observer/mod.rs +++ b/components/chainhook-event-observer/src/observer/mod.rs @@ -120,10 +120,8 @@ pub struct EventObserverConfig { pub control_port: u16, pub bitcoin_node_username: String, pub bitcoin_node_password: String, - pub bitcoin_node_rpc_host: String, - pub bitcoin_node_rpc_port: u16, - pub stacks_node_rpc_host: String, - pub stacks_node_rpc_port: u16, + pub bitcoin_node_rpc_url: String, + pub stacks_node_rpc_url: String, pub operators: HashSet, pub display_logs: bool, } @@ -192,8 +190,7 @@ pub struct BitcoinRPCRequest { pub struct BitcoinConfig { pub username: String, pub password: String, - pub rpc_host: String, - pub rpc_port: u16, + pub rpc_url: String, } #[derive(Debug, Clone)] @@ -216,14 +213,8 @@ pub async fn start_event_observer( info!("Event observer starting with config {:?}", config); let indexer = Indexer::new(IndexerConfig { - stacks_node_rpc_url: format!( - "{}:{}", - config.stacks_node_rpc_host, config.stacks_node_rpc_port - ), - bitcoin_node_rpc_url: format!( - "{}:{}", - config.bitcoin_node_rpc_host, config.bitcoin_node_rpc_port - ), + stacks_node_rpc_url: config.stacks_node_rpc_url.clone(), + bitcoin_node_rpc_url: config.bitcoin_node_rpc_url.clone(), bitcoin_node_rpc_username: config.bitcoin_node_username.clone(), bitcoin_node_rpc_password: config.bitcoin_node_password.clone(), stacks_network: StacksNetwork::Devnet, @@ -246,8 +237,7 @@ pub async fn start_event_observer( let bitcoin_config = BitcoinConfig { username: config.bitcoin_node_username.clone(), password: config.bitcoin_node_password.clone(), - rpc_host: config.bitcoin_node_rpc_host.clone(), - rpc_port: config.bitcoin_node_rpc_port, + rpc_url: config.bitcoin_node_rpc_url.clone(), }; let mut entries = HashMap::new(); @@ -456,10 +446,7 @@ pub async fn start_observer_commands_handler( } let bitcoin_client_rpc = Client::new( - &format!( - "{}:{}", - config.bitcoin_node_rpc_host, config.bitcoin_node_rpc_port - ), + &config.bitcoin_node_rpc_url, Auth::UserPass( config.bitcoin_node_username.to_string(), config.bitcoin_node_password.to_string(), @@ -1069,10 +1056,7 @@ pub async fn handle_bitcoin_rpc_call( let client = Client::new(); let builder = client - .post(format!( - "{}:{}", - bitcoin_config.rpc_host, bitcoin_config.rpc_port - )) + .post(&bitcoin_config.rpc_url) .header("Content-Type", "application/json") .timeout(std::time::Duration::from_secs(5)) .header("Authorization", format!("Basic {}", token)); diff --git a/components/chainhook-event-observer/src/observer/tests/mod.rs b/components/chainhook-event-observer/src/observer/tests/mod.rs index 5232adfd6..680e6817d 100644 --- a/components/chainhook-event-observer/src/observer/tests/mod.rs +++ b/components/chainhook-event-observer/src/observer/tests/mod.rs @@ -37,10 +37,8 @@ fn generate_test_config() -> (EventObserverConfig, ChainhookStore) { control_port: 0, bitcoin_node_username: "user".into(), bitcoin_node_password: "user".into(), - bitcoin_node_rpc_host: "http://localhost".into(), - bitcoin_node_rpc_port: 0, - stacks_node_rpc_host: "http://localhost".into(), - stacks_node_rpc_port: 0, + bitcoin_node_rpc_url: "http://localhost:20443".into(), + stacks_node_rpc_url: "http://localhost:18443".into(), operators, display_logs: false, }; diff --git a/components/chainhook-node/src/cli/mod.rs b/components/chainhook-node/src/cli/mod.rs index e46cdf0cb..e2b1886a8 100644 --- a/components/chainhook-node/src/cli/mod.rs +++ b/components/chainhook-node/src/cli/mod.rs @@ -268,10 +268,8 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply: control_port: DEFAULT_CONTROL_PORT, bitcoin_node_username: config.network.bitcoin_node_rpc_username.clone(), bitcoin_node_password: config.network.bitcoin_node_rpc_password.clone(), - bitcoin_node_rpc_host: config.network.bitcoin_node_rpc_url.clone(), - bitcoin_node_rpc_port: bitcoin_port, - stacks_node_rpc_host: "http://localhost".into(), - stacks_node_rpc_port: 20443, + bitcoin_node_rpc_url: config.network.bitcoin_node_rpc_url.clone(), + stacks_node_rpc_url: config.network.stacks_node_rpc_url.clone(), operators: HashSet::new(), display_logs: false, }; @@ -703,10 +701,8 @@ pub fn start_node(mut config: Config) { control_port: DEFAULT_CONTROL_PORT, bitcoin_node_username: config.network.bitcoin_node_rpc_username.clone(), bitcoin_node_password: config.network.bitcoin_node_rpc_password.clone(), - bitcoin_node_rpc_host: "http://localhost".into(), - bitcoin_node_rpc_port: 18443, - stacks_node_rpc_host: "http://localhost".into(), - stacks_node_rpc_port: 20443, + bitcoin_node_rpc_url: config.network.bitcoin_node_rpc_url.clone(), + stacks_node_rpc_url: config.network.stacks_node_rpc_url.clone(), operators: HashSet::new(), display_logs: false, }; diff --git a/components/chainhook-node/src/config/mod.rs b/components/chainhook-node/src/config/mod.rs index 7e438567b..d1702e6b5 100644 --- a/components/chainhook-node/src/config/mod.rs +++ b/components/chainhook-node/src/config/mod.rs @@ -249,7 +249,7 @@ impl Config { }, network: IndexerConfig { stacks_node_rpc_url: "http://0.0.0.0:20443".into(), - bitcoin_node_rpc_url: "http://0.0.0.0:18443".into(), + bitcoin_node_rpc_url: "http://0.0.0.0:18332".into(), bitcoin_node_rpc_username: "devnet".into(), bitcoin_node_rpc_password: "devnet".into(), stacks_network: StacksNetwork::Testnet, @@ -275,7 +275,7 @@ impl Config { }, network: IndexerConfig { stacks_node_rpc_url: "http://0.0.0.0:20443".into(), - bitcoin_node_rpc_url: "http://0.0.0.0:18443".into(), + bitcoin_node_rpc_url: "http://0.0.0.0:8332".into(), bitcoin_node_rpc_username: "devnet".into(), bitcoin_node_rpc_password: "devnet".into(), stacks_network: StacksNetwork::Mainnet, diff --git a/components/stacks-network/src/chains_coordinator.rs b/components/stacks-network/src/chains_coordinator.rs index bc13bd5df..5d5ce3cd7 100644 --- a/components/stacks-network/src/chains_coordinator.rs +++ b/components/stacks-network/src/chains_coordinator.rs @@ -95,10 +95,11 @@ impl DevnetEventObserverConfig { control_port: devnet_config.orchestrator_control_port, bitcoin_node_username: devnet_config.bitcoin_node_username.clone(), bitcoin_node_password: devnet_config.bitcoin_node_password.clone(), - bitcoin_node_rpc_host: "http://localhost".into(), - bitcoin_node_rpc_port: devnet_config.bitcoin_node_rpc_port, - stacks_node_rpc_host: "http://localhost".into(), - stacks_node_rpc_port: devnet_config.stacks_node_rpc_port, + bitcoin_node_rpc_url: format!( + "http://localhost:{}", + devnet_config.bitcoin_node_rpc_port + ), + stacks_node_rpc_url: format!("http://localhost:{}", devnet_config.stacks_node_rpc_port), operators: HashSet::new(), display_logs: true, }; From 7e5cbf28d14d6e458640a62eb819588fad32255a Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 11 Nov 2022 18:13:59 -0500 Subject: [PATCH 04/12] chore: introduce release feature for logs --- components/chainhook-node/Cargo.toml | 3 ++- components/chainhook-node/src/main.rs | 23 +++++++++++-------- .../components/chainhook-node.dockerfile | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/components/chainhook-node/Cargo.toml b/components/chainhook-node/Cargo.toml index dd0212348..d4e2dec9a 100644 --- a/components/chainhook-node/Cargo.toml +++ b/components/chainhook-node/Cargo.toml @@ -50,4 +50,5 @@ path = "src/lib.rs" [features] default = ["cli"] cli = ["clap", "clap_generate", "toml", "ctrlc", "slog-json", "slog-scope"] -debug = ["slog/max_level_trace", "slog/release_max_level_debug"] \ No newline at end of file +debug = ["slog/max_level_trace", "slog/release_max_level_debug"] +release = ["slog/max_level_info", "slog/release_max_level_info"] \ No newline at end of file diff --git a/components/chainhook-node/src/main.rs b/components/chainhook-node/src/main.rs index b5dc48a60..e958493a2 100644 --- a/components/chainhook-node/src/main.rs +++ b/components/chainhook-node/src/main.rs @@ -26,18 +26,21 @@ use slog_atomic::*; use std::sync::Mutex; fn main() { - let decorator = slog_term::TermDecorator::new().build(); - let drain = Mutex::new(slog_term::FullFormat::new(decorator).build()).fuse(); - let drain = slog_async::Async::new(drain).build().fuse(); - let drain = AtomicSwitch::new(drain); - - // Get a root logger that will log into a given drain. - // - // Note `o!` macro for more natural `OwnedKeyValue` sequence building. - let root = Logger::root(drain.fuse(), o!()); + let logger = if cfg!(feature = "release") { + slog::Logger::root( + Mutex::new(slog_json::Json::default(std::io::stderr())).map(slog::Fuse), + slog::o!("version" => env!("CARGO_PKG_VERSION")), + ) + } else { + let decorator = slog_term::TermDecorator::new().build(); + let drain = Mutex::new(slog_term::FullFormat::new(decorator).build()).fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + let drain = AtomicSwitch::new(drain); + Logger::root(drain.fuse(), o!()) + }; // slog_stdlog uses the logger from slog_scope, so set a logger there - let _guard = slog_scope::set_global_logger(root); + let _guard = slog_scope::set_global_logger(logger); cli::main(); } diff --git a/dockerfiles/components/chainhook-node.dockerfile b/dockerfiles/components/chainhook-node.dockerfile index 02aef3a6e..3bcf693bd 100644 --- a/dockerfiles/components/chainhook-node.dockerfile +++ b/dockerfiles/components/chainhook-node.dockerfile @@ -24,7 +24,7 @@ WORKDIR /src/components/chainhook-node RUN mkdir /out -RUN cargo build --release +RUN cargo build --features release --release RUN cp target/release/chainhook-node /out From 90b7fa12f7fd4cd5e09dbccc982deb63b0505edf Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Mon, 14 Nov 2022 13:58:16 -0500 Subject: [PATCH 05/12] fix: redis reconnect --- components/chainhook-node/src/cli/mod.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/components/chainhook-node/src/cli/mod.rs b/components/chainhook-node/src/cli/mod.rs index e2b1886a8..a00f482e2 100644 --- a/components/chainhook-node/src/cli/mod.rs +++ b/components/chainhook-node/src/cli/mod.rs @@ -724,16 +724,6 @@ pub fn start_node(mut config: Config) { let _ = hiro_system_kit::nestable_block_on(future); }); - let redis_config = config.expected_redis_config(); - let client = redis::Client::open(redis_config.uri.clone()).unwrap(); - let mut redis_con = match client.get_connection() { - Ok(con) => con, - Err(message) => { - error!("Redis: {}", message.to_string()); - panic!(); - } - }; - loop { let event = match observer_event_rx.recv() { Ok(cmd) => cmd, @@ -742,6 +732,15 @@ pub fn start_node(mut config: Config) { break; } }; + let redis_config = config.expected_redis_config(); + let client = redis::Client::open(redis_config.uri.clone()).unwrap(); + let mut redis_con = match client.get_connection() { + Ok(con) => con, + Err(message) => { + error!("Redis: {}", message.to_string()); + panic!(); + } + }; match event { ObserverEvent::HookRegistered(chain_hook) => { // If start block specified, use it. From 1e82a2cc05baf548ad75a48db36f05a7efcf02bc Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Mon, 14 Nov 2022 18:25:14 -0500 Subject: [PATCH 06/12] chore: remove unused imports --- .../src/chainhooks/bitcoin/mod.rs | 27 ++++++------------- .../src/chainhooks/stacks/mod.rs | 24 +++++------------ .../src/chainhooks/types.rs | 4 --- .../src/indexer/bitcoin/blocks_pool.rs | 7 ++--- .../src/indexer/bitcoin/mod.rs | 3 +-- .../src/indexer/bitcoin/tests.rs | 6 +---- .../src/indexer/mod.rs | 4 +-- .../src/indexer/stacks/blocks_pool.rs | 11 +++----- .../src/indexer/stacks/mod.rs | 3 +-- .../src/indexer/stacks/tests.rs | 5 +--- .../indexer/tests/helpers/bitcoin_blocks.rs | 5 ++-- .../src/indexer/tests/helpers/microblocks.rs | 5 ++-- .../indexer/tests/helpers/stacks_blocks.rs | 3 +-- .../indexer/tests/helpers/stacks_shapes.rs | 3 +-- .../src/indexer/tests/helpers/transactions.rs | 2 +- .../src/indexer/tests/mod.rs | 2 +- .../src/observer/mod.rs | 12 ++------- .../src/observer/tests/mod.rs | 12 ++++----- 18 files changed, 40 insertions(+), 98 deletions(-) diff --git a/components/chainhook-event-observer/src/chainhooks/bitcoin/mod.rs b/components/chainhook-event-observer/src/chainhooks/bitcoin/mod.rs index 5e11040d1..462439083 100644 --- a/components/chainhook-event-observer/src/chainhooks/bitcoin/mod.rs +++ b/components/chainhook-event-observer/src/chainhooks/bitcoin/mod.rs @@ -1,35 +1,24 @@ -use crate::utils::AbstractStacksBlock; - use super::types::{ - BitcoinChainhookSpecification, BitcoinPredicateType, BitcoinTransactionFilterPredicate, - ChainhookSpecification, ExactMatchingRule, HookAction, HookFormation, KeyRegistrationPredicate, - LockSTXPredicate, MatchingRule, PobPredicate, PoxPredicate, StacksChainhookSpecification, - StacksContractDeploymentPredicate, StacksTransactionFilterPredicate, TransferSTXPredicate, + BitcoinChainhookSpecification, BitcoinPredicateType, ExactMatchingRule, HookAction, + KeyRegistrationPredicate, LockSTXPredicate, MatchingRule, PobPredicate, PoxPredicate, + TransferSTXPredicate, }; use base58::FromBase58; use bitcoincore_rpc::bitcoin::blockdata::opcodes; use bitcoincore_rpc::bitcoin::blockdata::script::Builder as BitcoinScriptBuilder; use bitcoincore_rpc::bitcoin::util::address::Payload; -use bitcoincore_rpc::bitcoin::{Address, PubkeyHash, PublicKey, Script}; +use bitcoincore_rpc::bitcoin::Address; use chainhook_types::{ - BitcoinBlockData, BitcoinChainEvent, BitcoinTransactionData, BlockIdentifier, - StacksBaseChainOperation, StacksChainEvent, StacksNetwork, StacksTransactionData, - StacksTransactionEvent, StacksTransactionKind, TransactionIdentifier, + BitcoinBlockData, BitcoinChainEvent, BitcoinTransactionData, StacksBaseChainOperation, + TransactionIdentifier, }; -use clarity_repl::clarity::codec::StacksMessageCodec; -use clarity_repl::clarity::util::hash::{hex_bytes, to_hex, Hash160}; -use clarity_repl::clarity::vm::types::{CharType, SequenceData, Value as ClarityValue}; +use clarity_repl::clarity::util::hash::to_hex; use reqwest::{Client, Method}; -use serde::Serialize; use serde_json::Value as JsonValue; use std::collections::HashMap; -use std::io::Cursor; -use std::iter::Map; -use std::slice::Iter; use std::str::FromStr; -use reqwest::{Error, RequestBuilder, Response}; -use std::future::Future; +use reqwest::RequestBuilder; pub struct BitcoinTriggerChainhook<'a> { pub chainhook: &'a BitcoinChainhookSpecification, diff --git a/components/chainhook-event-observer/src/chainhooks/stacks/mod.rs b/components/chainhook-event-observer/src/chainhooks/stacks/mod.rs index 6ee1e2b0b..9766ef6db 100644 --- a/components/chainhook-event-observer/src/chainhooks/stacks/mod.rs +++ b/components/chainhook-event-observer/src/chainhooks/stacks/mod.rs @@ -1,34 +1,22 @@ use crate::utils::AbstractStacksBlock; use super::types::{ - BitcoinChainhookSpecification, BitcoinPredicateType, BitcoinTransactionFilterPredicate, - ChainhookSpecification, ExactMatchingRule, HookAction, HookFormation, KeyRegistrationPredicate, - LockSTXPredicate, MatchingRule, PobPredicate, PoxPredicate, StacksChainhookSpecification, - StacksContractDeploymentPredicate, StacksTransactionFilterPredicate, TransferSTXPredicate, + HookAction, StacksChainhookSpecification, StacksContractDeploymentPredicate, + StacksTransactionFilterPredicate, }; -use base58::FromBase58; -use bitcoincore_rpc::bitcoin::blockdata::opcodes; -use bitcoincore_rpc::bitcoin::blockdata::script::Builder as BitcoinScriptBuilder; -use bitcoincore_rpc::bitcoin::{Address, PubkeyHash, PublicKey, Script}; use chainhook_types::{ - BitcoinChainEvent, BitcoinTransactionData, BlockIdentifier, StacksBaseChainOperation, - StacksBlockData, StacksChainEvent, StacksNetwork, StacksTransactionData, - StacksTransactionEvent, StacksTransactionKind, TransactionIdentifier, + BlockIdentifier, StacksChainEvent, StacksTransactionData, StacksTransactionEvent, + StacksTransactionKind, TransactionIdentifier, }; use clarity_repl::clarity::codec::StacksMessageCodec; -use clarity_repl::clarity::util::hash::{hex_bytes, to_hex, Hash160}; +use clarity_repl::clarity::util::hash::hex_bytes; use clarity_repl::clarity::vm::types::{CharType, SequenceData, Value as ClarityValue}; use reqwest::{Client, Method}; -use serde::Serialize; use serde_json::Value as JsonValue; use std::collections::HashMap; use std::io::Cursor; -use std::iter::Map; -use std::slice::Iter; -use std::str::FromStr; -use reqwest::{Error, RequestBuilder, Response}; -use std::future::Future; +use reqwest::RequestBuilder; pub struct StacksTriggerChainhook<'a> { pub chainhook: &'a StacksChainhookSpecification, diff --git a/components/chainhook-event-observer/src/chainhooks/types.rs b/components/chainhook-event-observer/src/chainhooks/types.rs index 5a340fb3d..61cdc2c95 100644 --- a/components/chainhook-event-observer/src/chainhooks/types.rs +++ b/components/chainhook-event-observer/src/chainhooks/types.rs @@ -2,10 +2,6 @@ use clarity_repl::clarity::util::hash::hex_bytes; use reqwest::Url; use serde::ser::{SerializeSeq, Serializer}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; -use std::fs::File; -use std::io::{BufReader, Read}; -use std::path::PathBuf; use chainhook_types::{BitcoinNetwork, StacksNetwork}; diff --git a/components/chainhook-event-observer/src/indexer/bitcoin/blocks_pool.rs b/components/chainhook-event-observer/src/indexer/bitcoin/blocks_pool.rs index 3e7c791f9..c52e8a7a3 100644 --- a/components/chainhook-event-observer/src/indexer/bitcoin/blocks_pool.rs +++ b/components/chainhook-event-observer/src/indexer/bitcoin/blocks_pool.rs @@ -1,12 +1,9 @@ use crate::indexer::{ChainSegment, ChainSegmentIncompatibility}; -use crate::utils::AbstractBlock; -use bitcoincore_rpc::bitcoin::Block; use chainhook_types::{ BitcoinBlockData, BitcoinChainEvent, BitcoinChainUpdatedWithBlocksData, - BitcoinChainUpdatedWithReorgData, BitcoinTransactionData, BlockIdentifier, Chain, + BitcoinChainUpdatedWithReorgData, BlockIdentifier, }; -use clarity_repl::clarity::util::hash::to_hex; -use std::collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; pub struct BitcoinBlockPool { canonical_fork_id: usize, diff --git a/components/chainhook-event-observer/src/indexer/bitcoin/mod.rs b/components/chainhook-event-observer/src/indexer/bitcoin/mod.rs index d51c52470..d12e74a9a 100644 --- a/components/chainhook-event-observer/src/indexer/bitcoin/mod.rs +++ b/components/chainhook-event-observer/src/indexer/bitcoin/mod.rs @@ -7,14 +7,13 @@ use bitcoincore_rpc::bitcoin::hashes::Hash; use bitcoincore_rpc::bitcoin::{Block, BlockHash}; use bitcoincore_rpc::{Auth, Client, RpcApi}; pub use blocks_pool::BitcoinBlockPool; -use chainhook_types::bitcoin::{OutPoint, TxIn, TxOut, Witness}; +use chainhook_types::bitcoin::{OutPoint, TxIn, TxOut}; use chainhook_types::{ BitcoinBlockData, BitcoinBlockMetadata, BitcoinTransactionData, BitcoinTransactionMetadata, BlockCommitmentData, BlockIdentifier, KeyRegistrationData, LockSTXData, PobBlockCommitmentData, PoxBlockCommitmentData, PoxReward, StacksBaseChainOperation, TransactionIdentifier, TransferSTXData, }; -use clarity_repl::clarity::deps_common::bitcoin::blockdata::script::Script; use clarity_repl::clarity::util::hash::{hex_bytes, to_hex}; use rocket::serde::json::Value as JsonValue; diff --git a/components/chainhook-event-observer/src/indexer/bitcoin/tests.rs b/components/chainhook-event-observer/src/indexer/bitcoin/tests.rs index 0e3cd9f98..b43dcf47c 100644 --- a/components/chainhook-event-observer/src/indexer/bitcoin/tests.rs +++ b/components/chainhook-event-observer/src/indexer/bitcoin/tests.rs @@ -1,8 +1,4 @@ -use super::super::tests::{ - helpers, process_bitcoin_blocks_and_check_expectations, BitcoinChainEventExpectation, -}; -use super::BitcoinBlockPool; -use chainhook_types::{BitcoinBlockData, BitcoinChainEvent}; +use super::super::tests::{helpers, process_bitcoin_blocks_and_check_expectations}; #[test] fn test_bitcoin_vector_001() { diff --git a/components/chainhook-event-observer/src/indexer/mod.rs b/components/chainhook-event-observer/src/indexer/mod.rs index e6c5d1b7c..750dbcd73 100644 --- a/components/chainhook-event-observer/src/indexer/mod.rs +++ b/components/chainhook-event-observer/src/indexer/mod.rs @@ -3,9 +3,7 @@ pub mod stacks; use crate::utils::AbstractBlock; use chainhook_types::{ - BitcoinChainEvent, BitcoinNetwork, BlockIdentifier, StacksBlockData, StacksChainEvent, - StacksChainUpdatedWithBlocksData, StacksChainUpdatedWithMicroblocksData, - StacksMicroblocksTrail, StacksNetwork, + BitcoinChainEvent, BitcoinNetwork, BlockIdentifier, StacksChainEvent, StacksNetwork, }; use rocket::serde::json::Value as JsonValue; use stacks::StacksBlockPool; diff --git a/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs b/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs index 62c7764e3..ac3e50f89 100644 --- a/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs +++ b/components/chainhook-event-observer/src/indexer/stacks/blocks_pool.rs @@ -1,14 +1,11 @@ use crate::indexer::{ChainSegment, ChainSegmentIncompatibility}; -use crate::utils::AbstractBlock; -use bitcoincore_rpc::bitcoin::Block; use chainhook_types::{ - BitcoinChainEvent, BlockIdentifier, Chain, StacksBlockData, StacksBlockUpdate, - StacksChainEvent, StacksChainUpdatedWithBlocksData, StacksChainUpdatedWithMicroblocksData, + BlockIdentifier, StacksBlockData, StacksBlockUpdate, StacksChainEvent, + StacksChainUpdatedWithBlocksData, StacksChainUpdatedWithMicroblocksData, StacksChainUpdatedWithMicroblocksReorgData, StacksChainUpdatedWithReorgData, - StacksMicroblockData, StacksMicroblocksTrail, StacksTransactionData, + StacksMicroblockData, }; -use clarity_repl::clarity::util::hash::to_hex; -use std::collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet, VecDeque}; +use std::collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}; pub struct StacksBlockPool { canonical_fork_id: usize, diff --git a/components/chainhook-event-observer/src/indexer/stacks/mod.rs b/components/chainhook-event-observer/src/indexer/stacks/mod.rs index 099c64893..4fa1210b6 100644 --- a/components/chainhook-event-observer/src/indexer/stacks/mod.rs +++ b/components/chainhook-event-observer/src/indexer/stacks/mod.rs @@ -4,7 +4,6 @@ pub use blocks_pool::StacksBlockPool; use crate::indexer::AssetClassCache; use crate::indexer::{IndexerConfig, StacksChainContext}; -use bitcoincore_rpc::bitcoin::Block; use chainhook_types::*; use clarity_repl::clarity::codec::StacksMessageCodec; use clarity_repl::clarity::util::hash::hex_bytes; @@ -12,7 +11,7 @@ use clarity_repl::clarity::vm::types::Value as ClarityValue; use clarity_repl::codec::{StacksTransaction, TransactionAuth, TransactionPayload}; use rocket::serde::json::Value as JsonValue; use rocket::serde::Deserialize; -use std::collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::convert::TryInto; use std::io::Cursor; use std::str; diff --git a/components/chainhook-event-observer/src/indexer/stacks/tests.rs b/components/chainhook-event-observer/src/indexer/stacks/tests.rs index 9c3bde50d..1df4b685a 100644 --- a/components/chainhook-event-observer/src/indexer/stacks/tests.rs +++ b/components/chainhook-event-observer/src/indexer/stacks/tests.rs @@ -1,7 +1,4 @@ -use super::super::tests::{ - helpers, process_stacks_blocks_and_check_expectations, StacksChainEventExpectation, -}; -use super::StacksBlockPool; +use super::super::tests::{helpers, process_stacks_blocks_and_check_expectations}; #[test] fn test_stacks_vector_001() { diff --git a/components/chainhook-event-observer/src/indexer/tests/helpers/bitcoin_blocks.rs b/components/chainhook-event-observer/src/indexer/tests/helpers/bitcoin_blocks.rs index 0c7c8cd6f..3b16acb57 100644 --- a/components/chainhook-event-observer/src/indexer/tests/helpers/bitcoin_blocks.rs +++ b/components/chainhook-event-observer/src/indexer/tests/helpers/bitcoin_blocks.rs @@ -1,8 +1,7 @@ use chainhook_types::{ - BitcoinBlockData, BitcoinBlockMetadata, BitcoinTransactionData, BitcoinTransactionMetadata, - BlockIdentifier, + BitcoinBlockData, BitcoinBlockMetadata, BitcoinTransactionData, BlockIdentifier, }; -use clarity_repl::clarity::util::hash::{hex_bytes, to_hex}; +use clarity_repl::clarity::util::hash::to_hex; pub fn generate_test_bitcoin_block( fork_id: u8, diff --git a/components/chainhook-event-observer/src/indexer/tests/helpers/microblocks.rs b/components/chainhook-event-observer/src/indexer/tests/helpers/microblocks.rs index 2198bdb60..1d23ba75a 100644 --- a/components/chainhook-event-observer/src/indexer/tests/helpers/microblocks.rs +++ b/components/chainhook-event-observer/src/indexer/tests/helpers/microblocks.rs @@ -1,9 +1,8 @@ use super::BlockEvent; use chainhook_types::{ - BlockIdentifier, StacksBlockData, StacksMicroblockData, StacksMicroblockMetadata, - StacksTransactionData, StacksTransactionMetadata, + BlockIdentifier, StacksMicroblockData, StacksMicroblockMetadata, StacksTransactionData, }; -use clarity_repl::clarity::util::hash::{hex_bytes, to_hex}; +use clarity_repl::clarity::util::hash::to_hex; pub fn generate_test_microblock( fork_id: u8, diff --git a/components/chainhook-event-observer/src/indexer/tests/helpers/stacks_blocks.rs b/components/chainhook-event-observer/src/indexer/tests/helpers/stacks_blocks.rs index d02d9b5da..a841881f4 100644 --- a/components/chainhook-event-observer/src/indexer/tests/helpers/stacks_blocks.rs +++ b/components/chainhook-event-observer/src/indexer/tests/helpers/stacks_blocks.rs @@ -1,9 +1,8 @@ use super::BlockEvent; use chainhook_types::{ BlockIdentifier, StacksBlockData, StacksBlockMetadata, StacksTransactionData, - StacksTransactionMetadata, }; -use clarity_repl::clarity::util::hash::{hex_bytes, to_hex}; +use clarity_repl::clarity::util::hash::to_hex; pub fn generate_test_stacks_block( fork_id: u8, diff --git a/components/chainhook-event-observer/src/indexer/tests/helpers/stacks_shapes.rs b/components/chainhook-event-observer/src/indexer/tests/helpers/stacks_shapes.rs index 56205f329..d71c57415 100644 --- a/components/chainhook-event-observer/src/indexer/tests/helpers/stacks_shapes.rs +++ b/components/chainhook-event-observer/src/indexer/tests/helpers/stacks_shapes.rs @@ -1,7 +1,6 @@ use super::{super::StacksChainEventExpectation, BlockEvent}; use super::{microblocks, stacks_blocks}; -use bitcoincore_rpc::bitcoin::Block; -use chainhook_types::{StacksBlockData, StacksChainEvent, StacksMicroblockData}; +use chainhook_types::StacksChainEvent; pub fn expect_no_chain_update() -> StacksChainEventExpectation { Box::new(move |chain_event_to_check: Option| { diff --git a/components/chainhook-event-observer/src/indexer/tests/helpers/transactions.rs b/components/chainhook-event-observer/src/indexer/tests/helpers/transactions.rs index 9e14cdc21..0ce2543c7 100644 --- a/components/chainhook-event-observer/src/indexer/tests/helpers/transactions.rs +++ b/components/chainhook-event-observer/src/indexer/tests/helpers/transactions.rs @@ -9,7 +9,7 @@ use chainhook_types::{ StacksTransactionData, StacksTransactionKind, StacksTransactionMetadata, StacksTransactionReceipt, TransactionIdentifier, }; -use clarity_repl::clarity::util::hash::{hex_bytes, to_hex}; +use clarity_repl::clarity::util::hash::to_hex; use super::accounts; diff --git a/components/chainhook-event-observer/src/indexer/tests/mod.rs b/components/chainhook-event-observer/src/indexer/tests/mod.rs index 70e5d3c6c..c9a22a8e8 100644 --- a/components/chainhook-event-observer/src/indexer/tests/mod.rs +++ b/components/chainhook-event-observer/src/indexer/tests/mod.rs @@ -1,7 +1,7 @@ pub mod helpers; use self::helpers::BlockEvent; use super::{BitcoinBlockPool, StacksBlockPool}; -use chainhook_types::{BitcoinBlockData, BitcoinChainEvent, StacksBlockData, StacksChainEvent}; +use chainhook_types::{BitcoinBlockData, BitcoinChainEvent, StacksChainEvent}; pub type StacksChainEventExpectation = Box) -> ()>; diff --git a/components/chainhook-event-observer/src/observer/mod.rs b/components/chainhook-event-observer/src/observer/mod.rs index be4b3d13b..ebe3805cf 100644 --- a/components/chainhook-event-observer/src/observer/mod.rs +++ b/components/chainhook-event-observer/src/observer/mod.rs @@ -12,7 +12,7 @@ use bitcoincore_rpc::bitcoin::{BlockHash, Txid}; use bitcoincore_rpc::{Auth, Client, RpcApi}; use chainhook_types::{ BitcoinChainEvent, BitcoinNetwork, BlockIdentifier, StacksChainEvent, StacksNetwork, - StacksTransactionData, TransactionIdentifier, + TransactionIdentifier, }; use clarity_repl::clarity::util::hash::bytes_to_hex; use hiro_system_kit; @@ -20,24 +20,16 @@ use reqwest::Client as HttpClient; use rocket::config::{Config, LogLevel}; use rocket::data::{Limits, ToByteUnit}; use rocket::http::Status; -use rocket::outcome::IntoOutcome; use rocket::request::{self, FromRequest, Outcome, Request}; use rocket::serde::json::{json, Json, Value as JsonValue}; use rocket::serde::Deserialize; use rocket::State; use rocket_okapi::{openapi, openapi_get_routes, request::OpenApiFromRequest}; -use schemars::JsonSchema; -use serde_json::error; -use stacks_rpc_client::{PoxInfo, StacksRpc}; -use std::collections::{HashMap, HashSet, VecDeque}; -use std::convert::TryFrom; +use std::collections::{HashMap, HashSet}; use std::error::Error; -use std::iter::FromIterator; use std::net::{IpAddr, Ipv4Addr}; -use std::path::PathBuf; use std::str; use std::str::FromStr; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{Receiver, Sender}; use std::sync::{Arc, Mutex, RwLock}; diff --git a/components/chainhook-event-observer/src/observer/tests/mod.rs b/components/chainhook-event-observer/src/observer/tests/mod.rs index 680e6817d..68568fbfe 100644 --- a/components/chainhook-event-observer/src/observer/tests/mod.rs +++ b/components/chainhook-event-observer/src/observer/tests/mod.rs @@ -1,7 +1,7 @@ use crate::chainhooks::types::{ BitcoinChainhookSpecification, BitcoinPredicateType, BitcoinTransactionFilterPredicate, - ChainhookSpecification, ExactMatchingRule, HookAction, HookFormation, MatchingRule, Scope, - StacksBlockFilterPredicate, StacksChainhookSpecification, StacksContractCallBasedPredicate, + ChainhookSpecification, ExactMatchingRule, HookAction, HookFormation, Scope, + StacksChainhookSpecification, StacksContractCallBasedPredicate, StacksTransactionFilterPredicate, }; use crate::indexer::tests::helpers::transactions::generate_test_tx_bitcoin_p2pkh_transfer; @@ -9,14 +9,12 @@ use crate::indexer::tests::helpers::{ accounts, bitcoin_blocks, stacks_blocks, transactions::generate_test_tx_stacks_contract_call, }; use crate::observer::{ - self, start_observer_commands_handler, ApiKey, ChainhookStore, EventHandler, - EventObserverConfig, ObserverCommand, + start_observer_commands_handler, ApiKey, ChainhookStore, EventObserverConfig, ObserverCommand, }; use chainhook_types::{ - BitcoinChainEvent, BitcoinChainUpdatedWithBlocksData, BitcoinNetwork, StacksBlockData, - StacksBlockUpdate, StacksChainEvent, StacksChainUpdatedWithBlocksData, StacksNetwork, + BitcoinChainEvent, BitcoinChainUpdatedWithBlocksData, BitcoinNetwork, StacksBlockUpdate, + StacksChainEvent, StacksChainUpdatedWithBlocksData, StacksNetwork, }; -use clarity_repl::clarity::vm::types::QualifiedContractIdentifier; use hiro_system_kit; use std::collections::{HashMap, HashSet}; use std::sync::mpsc::{channel, Receiver, Sender}; From 285b87fe80a480286bd00b60125a1f34ace2f8a2 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Mon, 14 Nov 2022 18:25:59 -0500 Subject: [PATCH 07/12] refactor: centralize logging logic in hiro_system_kit --- Cargo.lock | 16 +++++------ .../chainhook-event-observer/Cargo.toml | 5 +--- .../chainhook-event-observer/src/main.rs | 26 +++++------------ components/chainhook-node/Cargo.toml | 16 ++--------- components/chainhook-node/src/cli/mod.rs | 12 ++++---- components/chainhook-node/src/lib.rs | 13 --------- components/chainhook-node/src/main.rs | 28 +------------------ components/hiro-system-kit/Cargo.toml | 12 +++++++- components/hiro-system-kit/src/lib.rs | 13 +++++++++ components/hiro-system-kit/src/log/mod.rs | 22 +++++++++++++++ 10 files changed, 71 insertions(+), 92 deletions(-) delete mode 100644 components/chainhook-node/src/lib.rs create mode 100644 components/hiro-system-kit/src/log/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 7a15d8d41..fa01731c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -785,9 +785,6 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "slog", - "slog-json", - "slog-scope", "stacks-rpc-client", "tokio", "toml 0.5.9", @@ -821,12 +818,6 @@ dependencies = [ "serde-redis", "serde_derive", "serde_json", - "slog", - "slog-async", - "slog-atomic", - "slog-json", - "slog-scope", - "slog-term", "tar", "tokio", "toml 0.5.9", @@ -2901,6 +2892,13 @@ dependencies = [ "ansi_term", "atty", "futures", + "lazy_static", + "slog", + "slog-async", + "slog-atomic", + "slog-json", + "slog-scope", + "slog-term", "tokio", ] diff --git a/components/chainhook-event-observer/Cargo.toml b/components/chainhook-event-observer/Cargo.toml index 11483421f..050e8b7cf 100644 --- a/components/chainhook-event-observer/Cargo.toml +++ b/components/chainhook-event-observer/Cargo.toml @@ -32,10 +32,7 @@ toml = { version = "0.5.6", features = ["preserve_order"], optional = true } ctrlc = { version = "3.2.2", optional = true } schemars = { version = "0.8.10" } rocket_okapi = "0.8.0-rc.1" -slog = { version = "2.7.0" } -slog-json = { version = "2.6.1", optional = true } -slog-scope = { version = "4.4.0", optional = true } [features] default = ["cli"] -cli = ["clap", "clap_generate", "toml", "ctrlc", "slog-json", "slog-scope"] +cli = ["clap", "clap_generate", "toml", "ctrlc", "hiro_system_kit/log"] diff --git a/components/chainhook-event-observer/src/main.rs b/components/chainhook-event-observer/src/main.rs index fb4b78b70..d3504a385 100644 --- a/components/chainhook-event-observer/src/main.rs +++ b/components/chainhook-event-observer/src/main.rs @@ -1,38 +1,31 @@ -#![allow(unused_imports)] - #[macro_use] extern crate rocket; -#[macro_use] -extern crate slog_scope; - -#[macro_use] -extern crate serde; - #[macro_use] extern crate serde_derive; #[macro_use] extern crate serde_json; +#[macro_use] +extern crate hiro_system_kit; + pub mod chainhooks; pub mod indexer; pub mod observer; pub mod utils; -use slog::Drain; -use std::sync::Mutex; +use hiro_system_kit::log::setup_global_logger; use crate::chainhooks::types::HookFormation; use clap::Parser; use ctrlc; use observer::{EventHandler, EventObserverConfig, ObserverCommand}; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::fs::File; use std::io::{BufReader, Read}; use std::path::PathBuf; -use std::sync::mpsc::{channel, Receiver, Sender}; -use toml::value::Value; +use std::sync::mpsc::channel; /// Simple program to greet a person #[derive(Parser, Debug)] @@ -45,13 +38,8 @@ struct Args { #[rocket::main] async fn main() { - let logger = slog::Logger::root( - Mutex::new(slog_json::Json::default(std::io::stderr())).map(slog::Fuse), - slog::o!("version" => env!("CARGO_PKG_VERSION")), - ); - // slog_stdlog uses the logger from slog_scope, so set a logger there - let _guard = slog_scope::set_global_logger(logger); + let _guard = setup_global_logger(); let args = Args::parse(); let config_path = get_config_path_or_exit(&args.config_path); diff --git a/components/chainhook-node/Cargo.toml b/components/chainhook-node/Cargo.toml index d4e2dec9a..a266de6ef 100644 --- a/components/chainhook-node/Cargo.toml +++ b/components/chainhook-node/Cargo.toml @@ -20,12 +20,6 @@ clap = { version = "3.2.23", features = ["derive"], optional = true } clap_generate = { version = "3.0.3", optional = true } toml = { version = "0.5.6", features = ["preserve_order"], optional = true } ctrlc = { version = "3.2.2", optional = true } -slog = { version = "2.7.0" } -slog-json = { version = "2.6.1", optional = true } -slog-scope = { version = "4.4.0", optional = true } -slog-term = "2.9.0" -slog-async = "2.7.0" -slog-atomic = "3.1.0" reqwest = { version = "0.11", features = ["stream", "json"] } tokio = { version = "=1.17.0", features = ["full"] } futures-util = "0.3.24" @@ -43,12 +37,8 @@ redis = "0.21.5" clarity_repl = { package = "clarity-repl", path = "../../components/clarity-repl" } hex = "0.4.3" -[lib] -name = "chainhook_db" -path = "src/lib.rs" - [features] default = ["cli"] -cli = ["clap", "clap_generate", "toml", "ctrlc", "slog-json", "slog-scope"] -debug = ["slog/max_level_trace", "slog/release_max_level_debug"] -release = ["slog/max_level_info", "slog/release_max_level_info"] \ No newline at end of file +cli = ["clap", "clap_generate", "toml", "ctrlc", "hiro_system_kit/log"] +debug = ["hiro_system_kit/debug"] +release = ["hiro_system_kit/release"] \ No newline at end of file diff --git a/components/chainhook-node/src/cli/mod.rs b/components/chainhook-node/src/cli/mod.rs index a00f482e2..28daa6e25 100644 --- a/components/chainhook-node/src/cli/mod.rs +++ b/components/chainhook-node/src/cli/mod.rs @@ -252,7 +252,7 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply: thread::spawn(move || { let res = block::digestion::start(digestion_rx, &digestion_config); if let Err(e) = res { - error!("{}", e); + crit!("{}", e); } let _ = terminate_observer_command_tx.send(ObserverCommand::Terminate); }); @@ -296,7 +296,7 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply: let mut redis_con = match client.get_connection() { Ok(con) => con, Err(message) => { - error!("Redis: {}", message.to_string()); + crit!("Redis: {}", message.to_string()); panic!(); } }; @@ -311,7 +311,7 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply: let event = match observer_event_rx.recv() { Ok(cmd) => cmd, Err(e) => { - error!("Error: broken channel {}", e.to_string()); + crit!("Error: broken channel {}", e.to_string()); break; } }; @@ -423,14 +423,14 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply: let start_block = match bitcoin_hook.start_block { Some(start_block) => start_block, None => { - error!("Bitcoin chainhook specification must include a field start_block in replay mode"); + warn!("Bitcoin chainhook specification must include a field start_block in replay mode"); continue; } }; let tip_height = match bitcoin_rpc.get_blockchain_info() { Ok(result) => result.blocks, Err(e) => { - error!("unable to retrieve Bitcoin chain tip ({})", e.to_string()); + warn!("unable to retrieve Bitcoin chain tip ({})", e.to_string()); continue; } }; @@ -898,7 +898,7 @@ fn update_storage_with_confirmed_stacks_blocks( ], ); if let Err(error) = res { - error!( + crit!( "unable to archive block {}: {}", block.block_identifier, error.to_string() diff --git a/components/chainhook-node/src/lib.rs b/components/chainhook-node/src/lib.rs deleted file mode 100644 index f18ca2829..000000000 --- a/components/chainhook-node/src/lib.rs +++ /dev/null @@ -1,13 +0,0 @@ -#[macro_use] -extern crate serde_json; - -#[macro_use] -extern crate slog_scope; - -#[macro_use] -extern crate serde_derive; - -extern crate serde; - -pub mod block; -pub mod config; diff --git a/components/chainhook-node/src/main.rs b/components/chainhook-node/src/main.rs index e958493a2..c365832ba 100644 --- a/components/chainhook-node/src/main.rs +++ b/components/chainhook-node/src/main.rs @@ -1,9 +1,6 @@ #[macro_use] extern crate serde_json; -#[macro_use] -extern crate slog_scope; - #[macro_use] extern crate hiro_system_kit; @@ -11,36 +8,13 @@ extern crate hiro_system_kit; extern crate serde_derive; extern crate serde; -extern crate slog; - -use slog_async; -use slog_term; pub mod archive; pub mod block; mod cli; pub mod config; -use slog::*; -use slog_atomic::*; -use std::sync::Mutex; - fn main() { - let logger = if cfg!(feature = "release") { - slog::Logger::root( - Mutex::new(slog_json::Json::default(std::io::stderr())).map(slog::Fuse), - slog::o!("version" => env!("CARGO_PKG_VERSION")), - ) - } else { - let decorator = slog_term::TermDecorator::new().build(); - let drain = Mutex::new(slog_term::FullFormat::new(decorator).build()).fuse(); - let drain = slog_async::Async::new(drain).build().fuse(); - let drain = AtomicSwitch::new(drain); - Logger::root(drain.fuse(), o!()) - }; - - // slog_stdlog uses the logger from slog_scope, so set a logger there - let _guard = slog_scope::set_global_logger(logger); - + let _guard = hiro_system_kit::log::setup_global_logger(); cli::main(); } diff --git a/components/hiro-system-kit/Cargo.toml b/components/hiro-system-kit/Cargo.toml index 707ee8d06..de5ea9f61 100644 --- a/components/hiro-system-kit/Cargo.toml +++ b/components/hiro-system-kit/Cargo.toml @@ -10,7 +10,17 @@ futures = "0.3.12" tokio = { version = "=1.17.0", optional = true } ansi_term = "0.12.1" atty = "0.2.14" +lazy_static = "1.4.0" +slog = { version = "2.7.0", optional = true } +slog-json = { version = "2.6.1", optional = true } +slog-scope = { version = "4.4.0", optional = true } +slog-term = { version = "2.9.0", optional = true } +slog-async = { version = "2.7.0", optional = true } +slog-atomic = { version = "3.1.0", optional = true } [features] default = ["tokio_helpers"] -tokio_helpers = ["tokio/full"] \ No newline at end of file +tokio_helpers = ["tokio/full"] +log = ["slog", "slog-json", "slog-scope", "slog-term", "slog-async", "slog-atomic"] +debug = ["log", "slog/max_level_trace", "slog/release_max_level_debug"] +release = ["log", "slog/max_level_info", "slog/release_max_level_info"] \ No newline at end of file diff --git a/components/hiro-system-kit/src/lib.rs b/components/hiro-system-kit/src/lib.rs index a3b0822c6..a14a16faa 100644 --- a/components/hiro-system-kit/src/lib.rs +++ b/components/hiro-system-kit/src/lib.rs @@ -1,10 +1,23 @@ mod macros; + #[cfg(feature = "tokio_helpers")] mod tokio_helpers; #[cfg(feature = "tokio_helpers")] pub use tokio_helpers::*; +#[cfg(feature = "log")] +pub mod log; + +#[cfg(feature = "log")] +pub extern crate slog_scope; + +#[cfg(feature = "log")] +pub use slog_scope::*; + +#[cfg(feature = "log")] +pub extern crate slog; + use std::thread::Builder; pub fn thread_named(name: &str) -> Builder { diff --git a/components/hiro-system-kit/src/log/mod.rs b/components/hiro-system-kit/src/log/mod.rs new file mode 100644 index 000000000..149663f12 --- /dev/null +++ b/components/hiro-system-kit/src/log/mod.rs @@ -0,0 +1,22 @@ +use slog::{o, Drain, Logger}; +use slog_async; +use slog_atomic::AtomicSwitch; +use slog_scope::GlobalLoggerGuard; +use slog_term; +use std::sync::Mutex; + +#[allow(dead_code)] +pub fn setup_global_logger() -> GlobalLoggerGuard { + slog_scope::set_global_logger(if cfg!(feature = "release") { + Logger::root( + Mutex::new(slog_json::Json::default(std::io::stderr())).map(slog::Fuse), + slog::o!(), + ) + } else { + let decorator = slog_term::TermDecorator::new().build(); + let drain = Mutex::new(slog_term::FullFormat::new(decorator).build()).fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + let drain = AtomicSwitch::new(drain); + Logger::root(drain.fuse(), o!()) + }) +} From 77661122f19b6190ba74c2ae81c583f738b4d095 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Tue, 15 Nov 2022 22:55:09 -0500 Subject: [PATCH 08/12] feat: update Stacks codec --- .../src/indexer/stacks/mod.rs | 6 +- .../src/observer/mod.rs | 24 +++++++- .../chainhook-node/src/block/digestion.rs | 2 +- .../chainhook-node/src/block/ingestion.rs | 2 +- .../clarinet-deployments/src/onchain/mod.rs | 10 ++-- components/clarity-repl/src/codec/mod.rs | 60 +++++++++++++++++-- 6 files changed, 89 insertions(+), 15 deletions(-) diff --git a/components/chainhook-event-observer/src/indexer/stacks/mod.rs b/components/chainhook-event-observer/src/indexer/stacks/mod.rs index 4fa1210b6..fcc4b0ea7 100644 --- a/components/chainhook-event-observer/src/indexer/stacks/mod.rs +++ b/components/chainhook-event-observer/src/indexer/stacks/mod.rs @@ -262,8 +262,8 @@ pub fn standardize_stacks_block( let (description, tx_type, fee, sender, sponsor) = match get_tx_description(&tx.raw_tx, &tx_events) { Ok(desc) => desc, - Err(_) => { - return Err(format!("unable to standardize block ({:?})", tx)); + Err(e) => { + return Err(format!("unable to standardize block ({})", e.to_string())); } }; let events = tx_events @@ -594,7 +594,7 @@ pub fn get_tx_description( }), ) } - TransactionPayload::SmartContract(ref smart_contract) => { + TransactionPayload::SmartContract(ref smart_contract, ref _clarity_version) => { let contract_identifier = format!("{}.{}", tx.origin_address(), smart_contract.name); let data = StacksContractDeploymentData { contract_identifier: contract_identifier.clone(), diff --git a/components/chainhook-event-observer/src/observer/mod.rs b/components/chainhook-event-observer/src/observer/mod.rs index ebe3805cf..29c533b66 100644 --- a/components/chainhook-event-observer/src/observer/mod.rs +++ b/components/chainhook-event-observer/src/observer/mod.rs @@ -275,6 +275,8 @@ pub async fn start_event_observer( handle_new_mempool_tx, handle_drop_mempool_tx, handle_new_attachement, + handle_mined_block, + handle_mined_microblock, ]; if bitcoin_rpc_proxy_enabled { @@ -900,7 +902,7 @@ pub fn handle_new_stacks_block( info!("unable to infer chain progress"); } Err(e) => { - error!("unable to handle stacks block: {}", e) + error!("{}", e) } } @@ -1024,6 +1026,26 @@ pub fn handle_new_attachement() -> Json { })) } +#[openapi(skip)] +#[post("/mined_block", format = "application/json", data = "")] +pub fn handle_mined_block(payload: Json) -> Json { + info!("POST /mined_block {:?}", payload); + Json(json!({ + "status": 200, + "result": "Ok", + })) +} + +#[openapi(skip)] +#[post("/mined_microblock", format = "application/json", data = "")] +pub fn handle_mined_microblock(payload: Json) -> Json { + info!("POST /mined_microblock {:?}", payload); + Json(json!({ + "status": 200, + "result": "Ok", + })) +} + #[openapi(skip)] #[post("/", format = "application/json", data = "")] pub async fn handle_bitcoin_rpc_call( diff --git a/components/chainhook-node/src/block/digestion.rs b/components/chainhook-node/src/block/digestion.rs index 2d070691b..331d49c42 100644 --- a/components/chainhook-node/src/block/digestion.rs +++ b/components/chainhook-node/src/block/digestion.rs @@ -46,7 +46,7 @@ pub fn start(command_rx: Receiver, config: &Config) -> Result< ) { Ok(block) => block, Err(e) => { - error!("unable to handle stacks block: {e}"); + error!("{e}"); continue; } }; diff --git a/components/chainhook-node/src/block/ingestion.rs b/components/chainhook-node/src/block/ingestion.rs index 8c35eface..091bfa690 100644 --- a/components/chainhook-node/src/block/ingestion.rs +++ b/components/chainhook-node/src/block/ingestion.rs @@ -108,7 +108,7 @@ pub fn start( ) { Ok(data) => data, Err(e) => { - error!("unable to handle stacks block: {e}"); + error!("{e}"); continue; } } diff --git a/components/clarinet-deployments/src/onchain/mod.rs b/components/clarinet-deployments/src/onchain/mod.rs index 8db39c9a4..c6b694b54 100644 --- a/components/clarinet-deployments/src/onchain/mod.rs +++ b/components/clarinet-deployments/src/onchain/mod.rs @@ -186,7 +186,7 @@ pub fn encode_contract_publish( }; sign_transaction_payload( account, - TransactionPayload::SmartContract(payload), + TransactionPayload::SmartContract(payload, None), nonce, tx_fee, anchor_mode, @@ -291,11 +291,13 @@ pub fn update_deployment_costs( }; } TransactionSpecification::ContractPublish(tx) => { - let transaction_payload = - TransactionPayload::SmartContract(TransactionSmartContract { + let transaction_payload = TransactionPayload::SmartContract( + TransactionSmartContract { name: tx.contract_name.clone(), code_body: StacksString::from_str(&tx.source).unwrap(), - }); + }, + None, + ); match stacks_rpc.estimate_transaction_fee(&transaction_payload, priority) { Ok(fee) => { diff --git a/components/clarity-repl/src/codec/mod.rs b/components/clarity-repl/src/codec/mod.rs index be116c92a..3475c9873 100644 --- a/components/clarity-repl/src/codec/mod.rs +++ b/components/clarity-repl/src/codec/mod.rs @@ -16,6 +16,7 @@ use clarity::util::retry::BoundReader; use clarity::util::secp256k1::{ MessageSignature, Secp256k1PrivateKey, Secp256k1PublicKey, MESSAGE_SIGNATURE_ENCODED_SIZE, }; +use clarity::vm::ClarityVersion; // use clarity::util::vrf::VRFProof; use clarity::vm::types::{ PrincipalData, QualifiedContractIdentifier, StandardPrincipalData, Value, @@ -1350,7 +1351,7 @@ impl_byte_array_serde!(TokenTransferMemo); pub enum TransactionPayload { TokenTransfer(PrincipalData, u64, TokenTransferMemo), ContractCall(TransactionContractCall), - SmartContract(TransactionSmartContract), + SmartContract(TransactionSmartContract, Option), PoisonMicroblock(StacksMicroblockHeader, StacksMicroblockHeader), // the previous epoch leader sent two microblocks with the same sequence, and this is proof Coinbase(CoinbasePayload, Option), } @@ -1376,6 +1377,7 @@ pub enum TransactionPayloadID { PoisonMicroblock = 3, Coinbase = 4, CoinbaseToAltRecipient = 5, + VersionedSmartContract = 6, } /// Encoding of an asset type identifier @@ -2380,6 +2382,29 @@ impl StacksMessageCodec for TransactionSmartContract { } } +fn ClarityVersion_consensus_serialize( + version: &ClarityVersion, + fd: &mut W, +) -> Result<(), CodecError> { + match *version { + ClarityVersion::Clarity1 => write_next(fd, &1u8)?, + ClarityVersion::Clarity2 => write_next(fd, &2u8)?, + } + Ok(()) +} + +fn ClarityVersion_consensus_deserialize(fd: &mut R) -> Result { + let version_byte: u8 = read_next(fd)?; + match version_byte { + 1u8 => Ok(ClarityVersion::Clarity1), + 2u8 => Ok(ClarityVersion::Clarity2), + _ => Err(CodecError::DeserializeError(format!( + "Unrecognized ClarityVersion byte {}", + &version_byte + ))), + } +} + impl StacksMessageCodec for TransactionPayload { fn consensus_serialize(&self, fd: &mut W) -> Result<(), CodecError> { match *self { @@ -2393,9 +2418,17 @@ impl StacksMessageCodec for TransactionPayload { write_next(fd, &(TransactionPayloadID::ContractCall as u8))?; cc.consensus_serialize(fd)?; } - TransactionPayload::SmartContract(ref sc) => { - write_next(fd, &(TransactionPayloadID::SmartContract as u8))?; - sc.consensus_serialize(fd)?; + TransactionPayload::SmartContract(ref sc, ref version_opt) => { + if let Some(version) = version_opt { + // caller requests a specific Clarity version + write_next(fd, &(TransactionPayloadID::VersionedSmartContract as u8))?; + ClarityVersion_consensus_serialize(&version, fd)?; + sc.consensus_serialize(fd)?; + } else { + // caller requests to use whatever the current clarity version is + write_next(fd, &(TransactionPayloadID::SmartContract as u8))?; + sc.consensus_serialize(fd)?; + } } _ => { unreachable!() @@ -2419,7 +2452,12 @@ impl StacksMessageCodec for TransactionPayload { } x if x == TransactionPayloadID::SmartContract as u8 => { let payload: TransactionSmartContract = read_next(fd)?; - TransactionPayload::SmartContract(payload) + TransactionPayload::SmartContract(payload, None) + } + x if x == TransactionPayloadID::VersionedSmartContract as u8 => { + let version = ClarityVersion_consensus_deserialize(fd)?; + let payload: TransactionSmartContract = read_next(fd)?; + TransactionPayload::SmartContract(payload, Some(version)) } x if x == TransactionPayloadID::PoisonMicroblock as u8 => { let micrblock1: StacksMicroblockHeader = read_next(fd)?; @@ -2430,6 +2468,18 @@ impl StacksMessageCodec for TransactionPayload { let payload: CoinbasePayload = read_next(fd)?; TransactionPayload::Coinbase(payload, None) } + x if x == TransactionPayloadID::CoinbaseToAltRecipient as u8 => { + let payload: CoinbasePayload = read_next(fd)?; + let principal_value: Value = read_next(fd)?; + let recipient = match principal_value { + Value::Principal(recipient_principal) => recipient_principal, + _ => { + return Err(CodecError::DeserializeError("Failed to parse coinbase transaction -- did not receive a recipient principal value".to_string())); + } + }; + + TransactionPayload::Coinbase(payload, Some(recipient)) + } _ => { return Err(CodecError::DeserializeError(format!( "Failed to parse transaction -- unknown payload ID {}", From f253307284e837e46ecd42f82d6094e7d0efdd3b Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Tue, 15 Nov 2022 22:56:17 -0500 Subject: [PATCH 09/12] feat: expose 2.1 settings --- .../clarinet-files/src/network_manifest.rs | 41 ++++- components/stacks-network/src/orchestrator.rs | 162 +++++++++++------- 2 files changed, 137 insertions(+), 66 deletions(-) diff --git a/components/clarinet-files/src/network_manifest.rs b/components/clarinet-files/src/network_manifest.rs index 3d28ebf9f..3fe79ec8b 100644 --- a/components/clarinet-files/src/network_manifest.rs +++ b/components/clarinet-files/src/network_manifest.rs @@ -16,6 +16,7 @@ use toml::value::Value; pub const DEFAULT_DERIVATION_PATH: &str = "m/44'/5757'/0'/0/0"; pub const DEFAULT_BITCOIN_NODE_IMAGE: &str = "quay.io/hirosystems/bitcoind:devnet-v2"; pub const DEFAULT_STACKS_NODE_IMAGE: &str = "quay.io/hirosystems/stacks-node:devnet-v2"; +pub const DEFAULT_STACKS_NODE_NEXT_IMAGE: &str = "quay.io/hirosystems/stacks-node:devnet-v3"; pub const DEFAULT_BITCOIN_EXPLORER_IMAGE: &str = "quay.io/hirosystems/bitcoin-explorer:devnet"; pub const DEFAULT_STACKS_API_IMAGE: &str = "blockstack/stacks-blockchain-api:latest"; pub const DEFAULT_STACKS_EXPLORER_IMAGE: &str = "hirosystems/explorer:latest"; @@ -32,6 +33,10 @@ pub const DEFAULT_DOCKER_SOCKET: &str = "npipe:////./pipe/docker_engine"; #[cfg(target_family = "wasm")] pub const DEFAULT_DOCKER_SOCKET: &str = "/var/run/docker.sock"; +pub const DEFAULT_EPOCH_2_0: u64 = 100; +pub const DEFAULT_EPOCH_2_05: u64 = 115; +pub const DEFAULT_EPOCH_2_1: u64 = 130; + #[derive(Serialize, Deserialize, Debug)] pub struct NetworkManifestFile { network: NetworkConfigFile, @@ -58,6 +63,9 @@ pub struct DevnetConfigFile { pub stacks_node_p2p_port: Option, pub stacks_node_rpc_port: Option, pub stacks_node_events_observers: Option>, + pub stacks_node_env_vars: Option>, + pub stacks_api_env_vars: Option>, + pub stacks_explorer_env_vars: Option>, pub stacks_api_port: Option, pub stacks_api_events_port: Option, pub bitcoin_explorer_port: Option, @@ -103,6 +111,10 @@ pub struct DevnetConfigFile { pub disable_subnet_api: Option, pub docker_host: Option, pub components_host: Option, + pub enable_next_features: Option, + pub epoch_2_0: Option, + pub epoch_2_05: Option, + pub epoch_2_1: Option, } #[derive(Serialize, Deserialize, Debug)] @@ -156,9 +168,12 @@ pub struct DevnetConfig { pub stacks_node_p2p_port: u16, pub stacks_node_rpc_port: u16, pub stacks_node_events_observers: Vec, + pub stacks_node_env_vars: Vec, pub stacks_api_port: u16, pub stacks_api_events_port: u16, + pub stacks_api_env_vars: Vec, pub stacks_explorer_port: u16, + pub stacks_explorer_env_vars: Vec, pub bitcoin_explorer_port: u16, pub bitcoin_controller_block_time: u32, pub bitcoin_controller_automining_disabled: bool, @@ -209,6 +224,10 @@ pub struct DevnetConfig { pub disable_subnet_api: bool, pub docker_host: String, pub components_host: String, + pub enable_next_features: bool, + pub epoch_2_0: u64, + pub epoch_2_05: u64, + pub epoch_2_1: u64, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -462,6 +481,7 @@ impl NetworkManifest { .expect("default deployer account unavailable"); let remapped_subnet_contract_id = format!("{}.{}", default_deployer.stx_address, contract_id.name); + let enable_next_features = devnet_config.enable_next_features.unwrap_or(false); let mut config = DevnetConfig { orchestrator_ingestion_port: devnet_config.orchestrator_port.unwrap_or(20445), @@ -525,10 +545,13 @@ impl NetworkManifest { .bitcoin_node_image_url .take() .unwrap_or(DEFAULT_BITCOIN_NODE_IMAGE.to_string()), - stacks_node_image_url: devnet_config - .stacks_node_image_url - .take() - .unwrap_or(DEFAULT_STACKS_NODE_IMAGE.to_string()), + stacks_node_image_url: devnet_config.stacks_node_image_url.take().unwrap_or( + match enable_next_features { + true => DEFAULT_STACKS_NODE_NEXT_IMAGE, + false => DEFAULT_STACKS_NODE_IMAGE, + } + .to_string(), + ), stacks_api_image_url: devnet_config .stacks_api_image_url .take() @@ -580,6 +603,16 @@ impl NetworkManifest { .docker_host .unwrap_or(DEFAULT_DOCKER_SOCKET.into()), components_host: devnet_config.components_host.unwrap_or("127.0.0.1".into()), + epoch_2_0: devnet_config.epoch_2_0.unwrap_or(DEFAULT_EPOCH_2_0), + epoch_2_05: devnet_config.epoch_2_05.unwrap_or(DEFAULT_EPOCH_2_05), + epoch_2_1: devnet_config.epoch_2_1.unwrap_or(DEFAULT_EPOCH_2_1), + stacks_node_env_vars: devnet_config.stacks_node_env_vars.take().unwrap_or(vec![]), + stacks_api_env_vars: devnet_config.stacks_api_env_vars.take().unwrap_or(vec![]), + stacks_explorer_env_vars: devnet_config + .stacks_explorer_env_vars + .take() + .unwrap_or(vec![]), + enable_next_features, }; if !config.disable_stacks_api && config.disable_stacks_api { config.disable_stacks_api = false; diff --git a/components/stacks-network/src/orchestrator.rs b/components/stacks-network/src/orchestrator.rs index fc14c0500..9b055baae 100644 --- a/components/stacks-network/src/orchestrator.rs +++ b/components/stacks-network/src/orchestrator.rs @@ -972,16 +972,6 @@ wait_time_for_microblocks = 5000 pox_sync_sample_secs = 10 microblock_frequency = 15000 -[burnchain] -chain = "bitcoin" -mode = "krypton" -poll_time_secs = 1 -peer_host = "host.docker.internal" -username = "{bitcoin_node_username}" -password = "{bitcoin_node_password}" -rpc_port = {orchestrator_ingestion_port} -peer_port = {bitcoin_node_p2p_port} - [miner] first_attempt_time_ms = 10000 subsequent_attempt_time_ms = 10000 @@ -990,10 +980,6 @@ subsequent_attempt_time_ms = 10000 stacks_node_rpc_port = devnet_config.stacks_node_rpc_port, stacks_node_p2p_port = devnet_config.stacks_node_p2p_port, miner_secret_key_hex = devnet_config.miner_secret_key_hex, - bitcoin_node_username = devnet_config.bitcoin_node_username, - bitcoin_node_password = devnet_config.bitcoin_node_password, - bitcoin_node_p2p_port = devnet_config.bitcoin_node_p2p_port, - orchestrator_ingestion_port = devnet_config.orchestrator_ingestion_port, ); for (_, account) in network_config.accounts.iter() { @@ -1048,6 +1034,50 @@ events_keys = ["*"] )); } + stacks_conf.push_str(&format!( + r#" +[burnchain] +chain = "bitcoin" +mode = "krypton" +poll_time_secs = 1 +peer_host = "host.docker.internal" +username = "{bitcoin_node_username}" +password = "{bitcoin_node_password}" +rpc_port = {orchestrator_ingestion_port} +peer_port = {bitcoin_node_p2p_port} +"#, + bitcoin_node_username = devnet_config.bitcoin_node_username, + bitcoin_node_password = devnet_config.bitcoin_node_password, + bitcoin_node_p2p_port = devnet_config.bitcoin_node_p2p_port, + orchestrator_ingestion_port = devnet_config.orchestrator_ingestion_port, + )); + + if devnet_config.enable_next_features { + stacks_conf.push_str(&format!( + r#"pox_2_activation = {epoch_2_1} + +[[burnchain.epochs]] +epoch_name = "1.0" +start_height = 0 + +[[burnchain.epochs]] +epoch_name = "2.0" +start_height = {epoch_2_0} + +[[burnchain.epochs]] +epoch_name = "2.05" +start_height = {epoch_2_05} + +[[burnchain.epochs]] +epoch_name = "2.1" +start_height = {epoch_2_1} + "#, + epoch_2_0 = devnet_config.epoch_2_0, + epoch_2_05 = devnet_config.epoch_2_05, + epoch_2_1 = devnet_config.epoch_2_1, + )); + } + let mut stacks_conf_path = PathBuf::from(&devnet_config.working_dir); stacks_conf_path.push("conf/Stacks.toml"); let mut file = File::create(stacks_conf_path) @@ -1088,6 +1118,12 @@ events_keys = ["*"] )) } + let mut env = vec![ + "STACKS_LOG_PP=1".to_string(), + "BLOCKSTACK_USE_TEST_GENESIS_CHAINSTATE=1".to_string(), + ]; + env.append(&mut devnet_config.stacks_node_env_vars.clone()); + let config = Config { labels: Some(labels), image: Some(devnet_config.stacks_node_image_url.clone()), @@ -1099,11 +1135,7 @@ events_keys = ["*"] "start".into(), "--config=/src/stacks-node/Stacks.toml".into(), ]), - env: Some(vec![ - "STACKS_LOG_PP=1".to_string(), - // "STACKS_LOG_DEBUG=1".to_string(), - "BLOCKSTACK_USE_TEST_GENESIS_CHAINSTATE=1".to_string(), - ]), + env: Some(env), host_config: Some(HostConfig { port_bindings: Some(port_bindings), binds: Some(binds), @@ -1510,39 +1542,42 @@ events_keys = ["*"] let mut labels = HashMap::new(); labels.insert("project".to_string(), self.network_name.to_string()); + let mut env = vec![ + format!("STACKS_CORE_RPC_HOST=stacks-node.{}", self.network_name), + format!("STACKS_BLOCKCHAIN_API_DB=pg"), + format!( + "STACKS_CORE_RPC_PORT={}", + devnet_config.stacks_node_rpc_port + ), + format!( + "STACKS_BLOCKCHAIN_API_PORT={}", + devnet_config.stacks_api_port + ), + format!("STACKS_BLOCKCHAIN_API_HOST=0.0.0.0"), + format!( + "STACKS_CORE_EVENT_PORT={}", + devnet_config.stacks_api_events_port + ), + format!("STACKS_CORE_EVENT_HOST=0.0.0.0"), + format!("STACKS_API_ENABLE_FT_METADATA=1"), + format!("PG_HOST=postgres.{}", self.network_name), + format!("PG_PORT=5432"), + format!("PG_USER={}", devnet_config.postgres_username), + format!("PG_PASSWORD={}", devnet_config.postgres_password), + format!("PG_DATABASE={}", devnet_config.stacks_api_postgres_database), + format!("STACKS_CHAIN_ID=2147483648"), + format!("V2_POX_MIN_AMOUNT_USTX=90000000260"), + "NODE_ENV=development".to_string(), + ]; + env.append(&mut devnet_config.stacks_api_env_vars.clone()); + let config = Config { labels: Some(labels), image: Some(devnet_config.stacks_api_image_url.clone()), domainname: Some(self.network_name.to_string()), tty: None, exposed_ports: Some(exposed_ports), - env: Some(vec![ - format!("STACKS_CORE_RPC_HOST=stacks-node.{}", self.network_name), - format!("STACKS_BLOCKCHAIN_API_DB=pg"), - format!( - "STACKS_CORE_RPC_PORT={}", - devnet_config.stacks_node_rpc_port - ), - format!( - "STACKS_BLOCKCHAIN_API_PORT={}", - devnet_config.stacks_api_port - ), - format!("STACKS_BLOCKCHAIN_API_HOST=0.0.0.0"), - format!( - "STACKS_CORE_EVENT_PORT={}", - devnet_config.stacks_api_events_port - ), - format!("STACKS_CORE_EVENT_HOST=0.0.0.0"), - format!("STACKS_API_ENABLE_FT_METADATA=1"), - format!("PG_HOST=postgres.{}", self.network_name), - format!("PG_PORT=5432"), - format!("PG_USER={}", devnet_config.postgres_username), - format!("PG_PASSWORD={}", devnet_config.postgres_password), - format!("PG_DATABASE={}", devnet_config.stacks_api_postgres_database), - format!("STACKS_CHAIN_ID=2147483648"), - format!("V2_POX_MIN_AMOUNT_USTX=90000000260"), - "NODE_ENV=development".to_string(), - ]), + env: Some(env), host_config: Some(HostConfig { port_bindings: Some(port_bindings), extra_hosts: Some(vec!["host.docker.internal:host-gateway".into()]), @@ -1912,28 +1947,31 @@ events_keys = ["*"] let mut labels = HashMap::new(); labels.insert("project".to_string(), self.network_name.to_string()); + let mut env = vec![ + format!( + "NEXT_PUBLIC_REGTEST_API_SERVER=http://localhost:{}", + devnet_config.stacks_api_port + ), + format!( + "NEXT_PUBLIC_TESTNET_API_SERVER=http://localhost:{}", + devnet_config.stacks_api_port + ), + format!( + "NEXT_PUBLIC_MAINNET_API_SERVER=http://localhost:{}", + devnet_config.stacks_api_port + ), + format!("NEXT_PUBLIC_DEFAULT_POLLING_INTERVAL={}", 5000), + "NODE_ENV=development".to_string(), + ]; + env.append(&mut devnet_config.stacks_node_env_vars.clone()); + let config = Config { labels: Some(labels), image: Some(devnet_config.stacks_explorer_image_url.clone()), domainname: Some(self.network_name.to_string()), tty: None, exposed_ports: Some(exposed_ports), - env: Some(vec![ - format!( - "NEXT_PUBLIC_REGTEST_API_SERVER=http://localhost:{}", - devnet_config.stacks_api_port - ), - format!( - "NEXT_PUBLIC_TESTNET_API_SERVER=http://localhost:{}", - devnet_config.stacks_api_port - ), - format!( - "NEXT_PUBLIC_MAINNET_API_SERVER=http://localhost:{}", - devnet_config.stacks_api_port - ), - format!("NEXT_PUBLIC_DEFAULT_POLLING_INTERVAL={}", 5000), - "NODE_ENV=development".to_string(), - ]), + env: Some(env), host_config: Some(HostConfig { port_bindings: Some(port_bindings), extra_hosts: Some(vec!["host.docker.internal:host-gateway".into()]), From 4c2417351d1cb4e5833b62f589d9439a52fc067b Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 16 Nov 2022 16:29:24 -0500 Subject: [PATCH 10/12] fix: pot pourri --- .../src/indexer/stacks/mod.rs | 4 +-- .../src/observer/mod.rs | 2 ++ components/chainhook-node/src/cli/mod.rs | 9 ++++++- .../stacks-network/src/chains_coordinator.rs | 27 ++++++++++++++++--- components/stacks-network/src/orchestrator.rs | 1 + 5 files changed, 36 insertions(+), 7 deletions(-) diff --git a/components/chainhook-event-observer/src/indexer/stacks/mod.rs b/components/chainhook-event-observer/src/indexer/stacks/mod.rs index fcc4b0ea7..d71ad18bf 100644 --- a/components/chainhook-event-observer/src/indexer/stacks/mod.rs +++ b/components/chainhook-event-observer/src/indexer/stacks/mod.rs @@ -246,8 +246,8 @@ pub fn standardize_stacks_block( let pox_cycle_length: u64 = (ctx.pox_info.prepare_phase_block_length + ctx.pox_info.reward_phase_block_length).into(); let current_len = block.burn_block_height - ctx.pox_info.first_burnchain_block_height; - let pox_cycle_id: u32 = (current_len / pox_cycle_length).try_into().unwrap(); - + let mut pox_cycle_id: u32 = (current_len / pox_cycle_length).try_into().unwrap(); + pox_cycle_id += 1; // Pox cycles are 1-indexed let mut events: HashMap<&String, Vec<&NewEvent>> = HashMap::new(); for event in block.events.iter() { events diff --git a/components/chainhook-event-observer/src/observer/mod.rs b/components/chainhook-event-observer/src/observer/mod.rs index 29c533b66..3ae017942 100644 --- a/components/chainhook-event-observer/src/observer/mod.rs +++ b/components/chainhook-event-observer/src/observer/mod.rs @@ -263,6 +263,7 @@ pub async fn start_event_observer( keep_alive: 5, temp_dir: std::env::temp_dir().into(), log_level: log_level.clone(), + cli_colors: false, limits, ..Config::default() }; @@ -301,6 +302,7 @@ pub async fn start_event_observer( keep_alive: 5, temp_dir: std::env::temp_dir().into(), log_level, + cli_colors: false, ..Config::default() }; diff --git a/components/chainhook-node/src/cli/mod.rs b/components/chainhook-node/src/cli/mod.rs index 28daa6e25..4ea99123a 100644 --- a/components/chainhook-node/src/cli/mod.rs +++ b/components/chainhook-node/src/cli/mod.rs @@ -305,7 +305,14 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply: config.network.bitcoin_node_rpc_username.clone(), config.network.bitcoin_node_rpc_password.clone(), ); - let bitcoin_rpc = Client::new(&config.network.bitcoin_node_rpc_url, auth).unwrap(); + + let bitcoin_rpc = match Client::new(&config.network.bitcoin_node_rpc_url, auth) { + Ok(con) => con, + Err(message) => { + crit!("Bitcoin RPC: {}", message.to_string()); + panic!(); + } + }; loop { let event = match observer_event_rx.recv() { diff --git a/components/stacks-network/src/chains_coordinator.rs b/components/stacks-network/src/chains_coordinator.rs index 5d5ce3cd7..e2ee42379 100644 --- a/components/stacks-network/src/chains_coordinator.rs +++ b/components/stacks-network/src/chains_coordinator.rs @@ -585,16 +585,35 @@ pub fn mine_bitcoin_block( ) { use bitcoincore_rpc::bitcoin::Address; use std::str::FromStr; - let rpc = Client::new( + let rpc = match Client::new( &format!("http://localhost:{}", bitcoin_node_rpc_port), Auth::UserPass( bitcoin_node_username.to_string(), bitcoin_node_password.to_string(), ), - ) - .unwrap(); + ) { + Ok(rpc) => rpc, + Err(e) => { + println!( + "{}: {}", + "unable to initialize bitcoin rpc client", + e.to_string() + ); + std::process::exit(1); + } + }; let miner_address = Address::from_str(miner_btc_address).unwrap(); - let _ = rpc.generate_to_address(1, &miner_address); + match rpc.generate_to_address(1, &miner_address) { + Ok(rpc) => rpc, + Err(e) => { + println!( + "{}: {}", + "unable to generate new bitcoin block", + e.to_string() + ); + std::process::exit(1); + } + }; } fn handle_bitcoin_mining( diff --git a/components/stacks-network/src/orchestrator.rs b/components/stacks-network/src/orchestrator.rs index 9b055baae..ff0638ad8 100644 --- a/components/stacks-network/src/orchestrator.rs +++ b/components/stacks-network/src/orchestrator.rs @@ -969,6 +969,7 @@ miner = true seed = "{miner_secret_key_hex}" local_peer_seed = "{miner_secret_key_hex}" wait_time_for_microblocks = 5000 +wait_time_for_blocks = 0 pox_sync_sample_secs = 10 microblock_frequency = 15000 From fa2be310b4b54f1a01c804b143806453e263330c Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 16 Nov 2022 18:04:21 -0500 Subject: [PATCH 11/12] chore: remove oreo-url from chainhook schema --- .../examples/cbtc/chainhooks/unwrap-btc.chainhook.yaml | 1 - .../examples/cbtc/chainhooks/wrap-btc.chainhook.yaml | 1 - components/clarinet-cli/src/chainhooks/types.rs | 1 - components/clarinet-cli/src/generate/chainhook.rs | 2 -- 4 files changed, 5 deletions(-) diff --git a/components/clarinet-cli/examples/cbtc/chainhooks/unwrap-btc.chainhook.yaml b/components/clarinet-cli/examples/cbtc/chainhooks/unwrap-btc.chainhook.yaml index 9dfdd2a8a..6cb97f618 100644 --- a/components/clarinet-cli/examples/cbtc/chainhooks/unwrap-btc.chainhook.yaml +++ b/components/clarinet-cli/examples/cbtc/chainhooks/unwrap-btc.chainhook.yaml @@ -4,7 +4,6 @@ version: 1 chain: stacks networks: devnet: - oreo-url: 'http://localhost:20446/' predicate: ft-event: asset-identifier: 'ST1PQHQKV0RJXZFY1DGX8MNSNYVE3VGZJSRTPGZGM.cbtc-token::cbtc' diff --git a/components/clarinet-cli/examples/cbtc/chainhooks/wrap-btc.chainhook.yaml b/components/clarinet-cli/examples/cbtc/chainhooks/wrap-btc.chainhook.yaml index 8e09e4246..895e3aa17 100644 --- a/components/clarinet-cli/examples/cbtc/chainhooks/wrap-btc.chainhook.yaml +++ b/components/clarinet-cli/examples/cbtc/chainhooks/wrap-btc.chainhook.yaml @@ -4,7 +4,6 @@ version: 1 chain: bitcoin networks: regtest: - oreo-url: 'http://localhost:20446/' predicate: scope: outputs p2wpkh: diff --git a/components/clarinet-cli/src/chainhooks/types.rs b/components/clarinet-cli/src/chainhooks/types.rs index e3a0dca56..e3bd49e7d 100644 --- a/components/clarinet-cli/src/chainhooks/types.rs +++ b/components/clarinet-cli/src/chainhooks/types.rs @@ -23,7 +23,6 @@ pub struct ChainhookNetworkSpecificationFile { expire_after_occurrence: Option, predicate: ChainhookPredicateFile, action: HookActionFile, - oreo_url: String, } #[derive(Debug, PartialEq, Serialize, Deserialize)] diff --git a/components/clarinet-cli/src/generate/chainhook.rs b/components/clarinet-cli/src/generate/chainhook.rs index aee3b3b6f..7b9389ec0 100644 --- a/components/clarinet-cli/src/generate/chainhook.rs +++ b/components/clarinet-cli/src/generate/chainhook.rs @@ -46,7 +46,6 @@ version: 1 chain: bitcoin networks: regtest: - oreo-url: http://localhost:20446/ predicate: confirmations: 1 # 1 to 7. 1 = optimistic and better UX except when the chain is forking tx-out: # support tx-in, tx-out. @@ -89,7 +88,6 @@ version: 1 chain: stacks networks: devnet: - oreo-url: http://localhost:20446/ predicate: print-event: contract-identifier: ST1PQHQKV0RJXZFY1DGX8MNSNYVE3VGZJSRTPGZGM.cbtc-token From 790c14bf8fd4a30d1f50c2c4a55873aecac1a076 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 16 Nov 2022 18:34:41 -0500 Subject: [PATCH 12/12] feat: deploy contracts using clarity_version --- components/clarinet-deployments/src/lib.rs | 14 ++++---- .../clarinet-deployments/src/onchain/mod.rs | 7 ++-- .../clarinet-deployments/src/requirements.rs | 32 +++++++++++++++---- components/clarinet-deployments/src/types.rs | 23 +++++++++++++ 4 files changed, 61 insertions(+), 15 deletions(-) diff --git a/components/clarinet-deployments/src/lib.rs b/components/clarinet-deployments/src/lib.rs index f26480af4..00c4ab1f2 100644 --- a/components/clarinet-deployments/src/lib.rs +++ b/components/clarinet-deployments/src/lib.rs @@ -352,12 +352,13 @@ pub async fn generate_default_deployment( Some(ast) => ast, None => { // Download the code - let (source, contract_location) = requirements::retrieve_contract( - &contract_id, - &cache_location, - &file_accessor, - ) - .await?; + let (source, clarity_version, contract_location) = + requirements::retrieve_contract( + &contract_id, + &cache_location, + &file_accessor, + ) + .await?; // Build the struct representing the requirement in the deployment if network.is_simnet() { @@ -394,6 +395,7 @@ pub async fn generate_default_deployment( location: contract_location, cost: deployment_fee_rate * source.len() as u64, remap_principals, + clarity_version, }; requirements_publish.insert(contract_id.clone(), data); } diff --git a/components/clarinet-deployments/src/onchain/mod.rs b/components/clarinet-deployments/src/onchain/mod.rs index c6b694b54..106ab8592 100644 --- a/components/clarinet-deployments/src/onchain/mod.rs +++ b/components/clarinet-deployments/src/onchain/mod.rs @@ -11,7 +11,7 @@ use clarity_repl::clarity::vm::types::{ PrincipalData, QualifiedContractIdentifier, StandardPrincipalData, }; use clarity_repl::clarity::vm::{ClarityName, Value}; -use clarity_repl::clarity::{ContractName, EvaluationResult}; +use clarity_repl::clarity::{ClarityVersion, ContractName, EvaluationResult}; use clarity_repl::codec::{ SinglesigHashMode, SinglesigSpendingCondition, StacksString, StacksTransactionSigner, TokenTransferMemo, TransactionAuth, TransactionContractCall, TransactionPayload, @@ -174,6 +174,7 @@ pub fn encode_stx_transfer( pub fn encode_contract_publish( contract_name: &ContractName, source: &str, + clarity_version: &ClarityVersion, account: &AccountConfig, nonce: u64, tx_fee: u64, @@ -186,7 +187,7 @@ pub fn encode_contract_publish( }; sign_transaction_payload( account, - TransactionPayload::SmartContract(payload, None), + TransactionPayload::SmartContract(payload, Some(clarity_version.clone())), nonce, tx_fee, anchor_mode, @@ -546,6 +547,7 @@ pub fn apply_on_chain_deployment( let transaction = match encode_contract_publish( &tx.contract_name, &source, + &tx.clarity_version, *account, nonce, tx.cost, @@ -621,6 +623,7 @@ pub fn apply_on_chain_deployment( let transaction = match encode_contract_publish( &tx.contract_id.name, &source, + &tx.clarity_version, *account, nonce, tx.cost, diff --git a/components/clarinet-deployments/src/requirements.rs b/components/clarinet-deployments/src/requirements.rs index 08c83db0e..b827dabc7 100644 --- a/components/clarinet-deployments/src/requirements.rs +++ b/components/clarinet-deployments/src/requirements.rs @@ -1,12 +1,12 @@ use clarinet_files::{FileAccessor, FileLocation}; -use clarity_repl::clarity::vm::types::QualifiedContractIdentifier; +use clarity_repl::clarity::{vm::types::QualifiedContractIdentifier, ClarityVersion}; use reqwest; pub async fn retrieve_contract( contract_id: &QualifiedContractIdentifier, cache_location: &FileLocation, file_accessor: &Option<&Box>, -) -> Result<(String, FileLocation), String> { +) -> Result<(String, ClarityVersion, FileLocation), String> { let contract_deployer = contract_id.issuer.to_address(); let contract_name = contract_id.name.to_string(); @@ -20,7 +20,11 @@ pub async fn retrieve_contract( }; if contract_source.is_ok() { - return Ok((contract_source.unwrap(), contract_location)); + return Ok(( + contract_source.unwrap(), + ClarityVersion::Clarity1, + contract_location, + )); } let stacks_node_addr = if contract_deployer.starts_with("SP") { @@ -36,19 +40,32 @@ pub async fn retrieve_contract( name = contract_name ); - let code = fetch_contract(request_url).await?.source; + let contract = fetch_contract(request_url).await?; let result = match file_accessor { - None => contract_location.write_content(code.as_bytes()), + None => contract_location.write_content(contract.source.as_bytes()), Some(file_accessor) => { file_accessor - .write_file(contract_location.to_string(), code.as_bytes()) + .write_file(contract_location.to_string(), contract.source.as_bytes()) .await } }; + let clarity_version = { + let version = contract.clarity_version.unwrap_or(1); + if version.eq(&1) { + ClarityVersion::Clarity1 + } else if version.eq(&2) { + ClarityVersion::Clarity2 + } else { + return Err(format!( + "unable to parse clarity_version (can either be '1' or '2'", + )); + } + }; + match result { - Ok(_) => Ok((code, contract_location)), + Ok(_) => Ok((contract.source, clarity_version, contract_location)), Err(err) => Err(err), } } @@ -58,6 +75,7 @@ pub async fn retrieve_contract( struct Contract { source: String, publish_height: u32, + clarity_version: Option, } async fn fetch_contract(request_url: String) -> Result { diff --git a/components/clarinet-deployments/src/types.rs b/components/clarinet-deployments/src/types.rs index 120b54a6e..18c449833 100644 --- a/components/clarinet-deployments/src/types.rs +++ b/components/clarinet-deployments/src/types.rs @@ -104,6 +104,8 @@ pub struct RequirementPublishSpecificationFile { pub path: Option, #[serde(skip_serializing_if = "Option::is_none")] pub url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub clarity_version: Option, } #[derive(Debug, PartialEq, Serialize, Deserialize)] @@ -387,6 +389,7 @@ pub struct RequirementPublishSpecification { pub remap_sender: StandardPrincipalData, pub remap_principals: BTreeMap, pub source: String, + pub clarity_version: ClarityVersion, pub cost: u64, pub location: FileLocation, } @@ -453,11 +456,27 @@ impl RequirementPublishSpecification { let source = location.read_content_as_utf8()?; + let clarity_version = match specs.clarity_version { + Some(clarity_version) => { + if clarity_version.eq(&1) { + Ok(ClarityVersion::Clarity1) + } else if clarity_version.eq(&2) { + Ok(ClarityVersion::Clarity2) + } else { + Err(format!( + "unable to parse clarity_version (can either be '1' or '2'", + )) + } + } + _ => Ok(DEFAULT_CLARITY_VERSION), + }?; + Ok(RequirementPublishSpecification { contract_id, remap_sender, remap_principals, source, + clarity_version, location: location, cost: specs.cost, }) @@ -954,6 +973,10 @@ impl TransactionPlanSpecification { path: None, url: None, cost: tx.cost, + clarity_version: match tx.clarity_version { + ClarityVersion::Clarity1 => Some(1), + ClarityVersion::Clarity2 => Some(2), + }, }, ) }