Merge pull request #669 from hirosystems/feat/stacks-2-1-support

Stacks 2.1 support

components/chainhook-event-observer/Cargo.toml

@@ -32,10 +32,7 @@ toml = { version = "0.5.6", features = ["preserve_order"], optional = true }
 ctrlc = { version = "3.2.2", optional = true }
 schemars = { version = "0.8.10" }
 rocket_okapi = "0.8.0-rc.1"
-slog = { version = "2.7.0" }
-slog-json = { version = "2.6.1", optional = true }
-slog-scope = { version = "4.4.0", optional = true }
 
 [features]
 default = ["cli"]
-cli = ["clap", "clap_generate", "toml", "ctrlc", "slog-json", "slog-scope"]
+cli = ["clap", "clap_generate", "toml", "ctrlc", "hiro_system_kit/log"]

components/chainhook-event-observer/src/chainhooks/bitcoin/mod.rs

@@ -1,35 +1,24 @@
-use crate::utils::AbstractStacksBlock;
-
 use super::types::{
-    BitcoinChainhookSpecification, BitcoinPredicateType, BitcoinTransactionFilterPredicate,
-    ChainhookSpecification, ExactMatchingRule, HookAction, HookFormation, KeyRegistrationPredicate,
-    LockSTXPredicate, MatchingRule, PobPredicate, PoxPredicate, StacksChainhookSpecification,
-    StacksContractDeploymentPredicate, StacksTransactionFilterPredicate, TransferSTXPredicate,
+    BitcoinChainhookSpecification, BitcoinPredicateType, ExactMatchingRule, HookAction,
+    KeyRegistrationPredicate, LockSTXPredicate, MatchingRule, PobPredicate, PoxPredicate,
+    TransferSTXPredicate,
 };
 use base58::FromBase58;
 use bitcoincore_rpc::bitcoin::blockdata::opcodes;
 use bitcoincore_rpc::bitcoin::blockdata::script::Builder as BitcoinScriptBuilder;
 use bitcoincore_rpc::bitcoin::util::address::Payload;
-use bitcoincore_rpc::bitcoin::{Address, PubkeyHash, PublicKey, Script};
+use bitcoincore_rpc::bitcoin::Address;
 use chainhook_types::{
-    BitcoinBlockData, BitcoinChainEvent, BitcoinTransactionData, BlockIdentifier,
-    StacksBaseChainOperation, StacksChainEvent, StacksNetwork, StacksTransactionData,
-    StacksTransactionEvent, StacksTransactionKind, TransactionIdentifier,
+    BitcoinBlockData, BitcoinChainEvent, BitcoinTransactionData, StacksBaseChainOperation,
+    TransactionIdentifier,
 };
-use clarity_repl::clarity::codec::StacksMessageCodec;
-use clarity_repl::clarity::util::hash::{hex_bytes, to_hex, Hash160};
-use clarity_repl::clarity::vm::types::{CharType, SequenceData, Value as ClarityValue};
+use clarity_repl::clarity::util::hash::to_hex;
 use reqwest::{Client, Method};
-use serde::Serialize;
 use serde_json::Value as JsonValue;
 use std::collections::HashMap;
-use std::io::Cursor;
-use std::iter::Map;
-use std::slice::Iter;
 use std::str::FromStr;
 
-use reqwest::{Error, RequestBuilder, Response};
-use std::future::Future;
+use reqwest::RequestBuilder;
 
 pub struct BitcoinTriggerChainhook<'a> {
     pub chainhook: &'a BitcoinChainhookSpecification,

components/chainhook-event-observer/src/chainhooks/stacks/mod.rs

@@ -1,34 +1,22 @@
 use crate::utils::AbstractStacksBlock;
 
 use super::types::{
-    BitcoinChainhookSpecification, BitcoinPredicateType, BitcoinTransactionFilterPredicate,
-    ChainhookSpecification, ExactMatchingRule, HookAction, HookFormation, KeyRegistrationPredicate,
-    LockSTXPredicate, MatchingRule, PobPredicate, PoxPredicate, StacksChainhookSpecification,
-    StacksContractDeploymentPredicate, StacksTransactionFilterPredicate, TransferSTXPredicate,
+    HookAction, StacksChainhookSpecification, StacksContractDeploymentPredicate,
+    StacksTransactionFilterPredicate,
 };
-use base58::FromBase58;
-use bitcoincore_rpc::bitcoin::blockdata::opcodes;
-use bitcoincore_rpc::bitcoin::blockdata::script::Builder as BitcoinScriptBuilder;
-use bitcoincore_rpc::bitcoin::{Address, PubkeyHash, PublicKey, Script};
 use chainhook_types::{
-    BitcoinChainEvent, BitcoinTransactionData, BlockIdentifier, StacksBaseChainOperation,
-    StacksBlockData, StacksChainEvent, StacksNetwork, StacksTransactionData,
-    StacksTransactionEvent, StacksTransactionKind, TransactionIdentifier,
+    BlockIdentifier, StacksChainEvent, StacksTransactionData, StacksTransactionEvent,
+    StacksTransactionKind, TransactionIdentifier,
 };
 use clarity_repl::clarity::codec::StacksMessageCodec;
-use clarity_repl::clarity::util::hash::{hex_bytes, to_hex, Hash160};
+use clarity_repl::clarity::util::hash::hex_bytes;
 use clarity_repl::clarity::vm::types::{CharType, SequenceData, Value as ClarityValue};
 use reqwest::{Client, Method};
-use serde::Serialize;
 use serde_json::Value as JsonValue;
 use std::collections::HashMap;
 use std::io::Cursor;
-use std::iter::Map;
-use std::slice::Iter;
-use std::str::FromStr;
 
-use reqwest::{Error, RequestBuilder, Response};
-use std::future::Future;
+use reqwest::RequestBuilder;
 
 pub struct StacksTriggerChainhook<'a> {
     pub chainhook: &'a StacksChainhookSpecification,

components/chainhook-event-observer/src/chainhooks/types.rs

@@ -2,10 +2,6 @@ use clarity_repl::clarity::util::hash::hex_bytes;
 use reqwest::Url;
 use serde::ser::{SerializeSeq, Serializer};
 use serde::{Deserialize, Serialize};
-use std::collections::BTreeMap;
-use std::fs::File;
-use std::io::{BufReader, Read};
-use std::path::PathBuf;
 
 use chainhook_types::{BitcoinNetwork, StacksNetwork};
 

components/chainhook-event-observer/src/indexer/bitcoin/blocks_pool.rs

@@ -1,12 +1,9 @@
 use crate::indexer::{ChainSegment, ChainSegmentIncompatibility};
-use crate::utils::AbstractBlock;
-use bitcoincore_rpc::bitcoin::Block;
 use chainhook_types::{
     BitcoinBlockData, BitcoinChainEvent, BitcoinChainUpdatedWithBlocksData,
-    BitcoinChainUpdatedWithReorgData, BitcoinTransactionData, BlockIdentifier, Chain,
+    BitcoinChainUpdatedWithReorgData, BlockIdentifier,
 };
-use clarity_repl::clarity::util::hash::to_hex;
-use std::collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 
 pub struct BitcoinBlockPool {
     canonical_fork_id: usize,
@@ -30,7 +27,7 @@ impl BitcoinBlockPool {
     pub fn process_block(
         &mut self,
         block: BitcoinBlockData,
-    ) -> Result<Option<BitcoinChainEvent>, ()> {
+    ) -> Result<Option<BitcoinChainEvent>, String> {
         info!("Start processing Bitcoin {}", block.block_identifier);
 
         // Keep block data in memory
@@ -52,7 +49,10 @@ impl BitcoinBlockPool {
         let previous_canonical_fork_id = self.canonical_fork_id;
         let previous_canonical_fork = match self.forks.get(&previous_canonical_fork_id) {
             Some(fork) => fork.clone(),
-            None => return Err(()),
+            None => {
+                error!("unable to retrieve previous bitcoin fork");
+                return Ok(None);
+            }
         };
 
         let mut fork_updated = None;

components/chainhook-event-observer/src/indexer/bitcoin/mod.rs

@@ -7,14 +7,13 @@ use bitcoincore_rpc::bitcoin::hashes::Hash;
 use bitcoincore_rpc::bitcoin::{Block, BlockHash};
 use bitcoincore_rpc::{Auth, Client, RpcApi};
 pub use blocks_pool::BitcoinBlockPool;
-use chainhook_types::bitcoin::{OutPoint, TxIn, TxOut, Witness};
+use chainhook_types::bitcoin::{OutPoint, TxIn, TxOut};
 use chainhook_types::{
     BitcoinBlockData, BitcoinBlockMetadata, BitcoinTransactionData, BitcoinTransactionMetadata,
     BlockCommitmentData, BlockIdentifier, KeyRegistrationData, LockSTXData, PobBlockCommitmentData,
     PoxBlockCommitmentData, PoxReward, StacksBaseChainOperation, TransactionIdentifier,
     TransferSTXData,
 };
-use clarity_repl::clarity::deps_common::bitcoin::blockdata::script::Script;
 use clarity_repl::clarity::util::hash::{hex_bytes, to_hex};
 use rocket::serde::json::Value as JsonValue;
 
@@ -37,24 +36,30 @@ pub struct RewardParticipant {
 pub fn standardize_bitcoin_block(
     indexer_config: &IndexerConfig,
     marshalled_block: JsonValue,
-) -> BitcoinBlockData {
+) -> Result<BitcoinBlockData, String> {
     let auth = Auth::UserPass(
         indexer_config.bitcoin_node_rpc_username.clone(),
         indexer_config.bitcoin_node_rpc_password.clone(),
     );
-
-    let rpc = Client::new(&indexer_config.bitcoin_node_rpc_url, auth).unwrap();
-
-    let partial_block: NewBitcoinBlock = serde_json::from_value(marshalled_block).unwrap();
+    let rpc = Client::new(&indexer_config.bitcoin_node_rpc_url, auth).map_err(|e| {
+        format!(
+            "unable for bitcoin rpc initialize client: {}",
+            e.to_string()
+        )
+    })?;
+    let partial_block: NewBitcoinBlock = serde_json::from_value(marshalled_block)
+        .map_err(|e| format!("unable for parse bitcoin block: {}", e.to_string()))?;
     let block_hash = {
         let block_hash_str = partial_block.burn_block_hash.strip_prefix("0x").unwrap();
         let mut block_hash_bytes = hex_bytes(&block_hash_str).unwrap();
         block_hash_bytes.reverse();
         BlockHash::from_slice(&block_hash_bytes).unwrap()
     };
-    let block = rpc.get_block(&block_hash).unwrap();
+    let block = rpc
+        .get_block(&block_hash)
+        .map_err(|e| format!("unable for invoke rpc get_block: {}", e.to_string()))?;
     let block_height = partial_block.burn_block_height;
-    build_block(block, block_height, indexer_config)
+    Ok(build_block(block, block_height, indexer_config))
 }
 
 pub fn build_block(

components/chainhook-event-observer/src/indexer/bitcoin/tests.rs

@@ -1,8 +1,4 @@
-use super::super::tests::{
-    helpers, process_bitcoin_blocks_and_check_expectations, BitcoinChainEventExpectation,
-};
-use super::BitcoinBlockPool;
-use chainhook_types::{BitcoinBlockData, BitcoinChainEvent};
+use super::super::tests::{helpers, process_bitcoin_blocks_and_check_expectations};
 
 #[test]
 fn test_bitcoin_vector_001() {

components/chainhook-event-observer/src/indexer/mod.rs

@@ -3,9 +3,7 @@ pub mod stacks;
 
 use crate::utils::AbstractBlock;
 use chainhook_types::{
-    BitcoinChainEvent, BitcoinNetwork, BlockIdentifier, StacksBlockData, StacksChainEvent,
-    StacksChainUpdatedWithBlocksData, StacksChainUpdatedWithMicroblocksData,
-    StacksMicroblocksTrail, StacksNetwork,
+    BitcoinChainEvent, BitcoinNetwork, BlockIdentifier, StacksChainEvent, StacksNetwork,
 };
 use rocket::serde::json::Value as JsonValue;
 use stacks::StacksBlockPool;
@@ -67,56 +65,57 @@ impl Indexer {
     pub fn handle_bitcoin_block(
         &mut self,
         marshalled_block: JsonValue,
-    ) -> Result<Option<BitcoinChainEvent>, ()> {
-        let block = bitcoin::standardize_bitcoin_block(&self.config, marshalled_block);
-        self.bitcoin_blocks_pool.process_block(block)
+    ) -> Result<Option<BitcoinChainEvent>, String> {
+        let block = bitcoin::standardize_bitcoin_block(&self.config, marshalled_block)?;
+        let event = self.bitcoin_blocks_pool.process_block(block);
+        event
     }
 
     pub fn handle_stacks_serialized_block(
         &mut self,
         serialized_block: &str,
-    ) -> Result<Option<StacksChainEvent>, ()> {
+    ) -> Result<Option<StacksChainEvent>, String> {
         let block = stacks::standardize_stacks_serialized_block(
             &self.config,
             serialized_block,
             &mut self.stacks_context,
-        );
+        )?;
         self.stacks_blocks_pool.process_block(block)
     }
 
     pub fn handle_stacks_marshalled_block(
         &mut self,
         marshalled_block: JsonValue,
-    ) -> Result<Option<StacksChainEvent>, ()> {
+    ) -> Result<Option<StacksChainEvent>, String> {
         let block = stacks::standardize_stacks_marshalled_block(
             &self.config,
             marshalled_block,
             &mut self.stacks_context,
-        );
+        )?;
         self.stacks_blocks_pool.process_block(block)
     }
 
     pub fn handle_stacks_serialized_microblock_trail(
         &mut self,
         serialized_microblock_trail: &str,
-    ) -> Option<StacksChainEvent> {
+    ) -> Result<Option<StacksChainEvent>, String> {
         let microblocks = stacks::standardize_stacks_serialized_microblock_trail(
             &self.config,
             serialized_microblock_trail,
             &mut self.stacks_context,
-        );
+        )?;
         self.stacks_blocks_pool.process_microblocks(microblocks)
     }
 
     pub fn handle_stacks_marshalled_microblock_trail(
         &mut self,
         marshalled_microblock_trail: JsonValue,
-    ) -> Option<StacksChainEvent> {
+    ) -> Result<Option<StacksChainEvent>, String> {
         let microblocks = stacks::standardize_stacks_marshalled_microblock_trail(
             &self.config,
             marshalled_microblock_trail,
             &mut self.stacks_context,
-        );
+        )?;
         self.stacks_blocks_pool.process_microblocks(microblocks)
     }