Feat/shadow block tooling

.github/actions/dockerfiles/Dockerfile.debian-source

@@ -24,5 +24,5 @@ RUN --mount=type=tmpfs,target=${BUILD_DIR} cp -R /src/. ${BUILD_DIR}/ \
     && cp -R ${BUILD_DIR}/target/${TARGET}/release/. /out
 
 FROM --platform=${TARGETPLATFORM} debian:bookworm
-COPY --from=build /out/stacks-node /out/stacks-signer /bin/
+COPY --from=build /out/stacks-node /out/stacks-signer /out/stacks-inspect /bin/
 CMD ["stacks-node", "mainnet"]

.github/workflows/bitcoin-tests.yml

@@ -133,6 +133,7 @@ jobs:
           - tests::nakamoto_integrations::utxo_check_on_startup_panic
           - tests::nakamoto_integrations::utxo_check_on_startup_recover
           - tests::nakamoto_integrations::v3_signer_api_endpoint
+          - tests::nakamoto_integrations::test_shadow_recovery
           # TODO: enable these once v1 signer is supported by a new nakamoto epoch
           # - tests::signer::v1::dkg
           # - tests::signer::v1::sign_request_rejected

stackslib/src/burnchains/mod.rs

@@ -450,6 +450,7 @@ impl PoxConstants {
         )
     }
 
+    // NOTE: this is the *old* pre-Nakamoto testnet
     pub fn testnet_default() -> PoxConstants {
         PoxConstants::new(
             POX_REWARD_CYCLE_LENGTH / 2,   // 1050

stackslib/src/chainstate/nakamoto/coordinator/mod.rs

@@ -58,6 +58,9 @@ use crate::monitoring::increment_stx_blocks_processed_counter;
 use crate::net::Error as NetError;
 use crate::util_lib::db::Error as DBError;
 
+#[cfg(any(test, feature = "testing"))]
+pub static TEST_COORDINATOR_STALL: std::sync::Mutex<Option<bool>> = std::sync::Mutex::new(None);
+
 #[cfg(test)]
 pub mod tests;
 
@@ -764,6 +767,21 @@ impl<
         true
     }
 
+    #[cfg(any(test, feature = "testing"))]
+    fn fault_injection_pause_nakamoto_block_processing() {
+        if *TEST_COORDINATOR_STALL.lock().unwrap() == Some(true) {
+            // Do an extra check just so we don't log EVERY time.
+            warn!("Coordinator is stalled due to testing directive");
+            while *TEST_COORDINATOR_STALL.lock().unwrap() == Some(true) {
+                std::thread::sleep(std::time::Duration::from_millis(10));
+            }
+            warn!("Coordinator is no longer stalled due to testing directive. Continuing...");
+        }
+    }
+
+    #[cfg(not(any(test, feature = "testing")))]
+    fn fault_injection_pause_nakamoto_block_processing() {}
+
     /// Handle one or more new Nakamoto Stacks blocks.
     /// If we process a PoX anchor block, then return its block hash.  This unblocks processing the
     /// next reward cycle's burnchain blocks.  Subsequent calls to this function will terminate
@@ -776,6 +794,8 @@ impl<
         );
 
         loop {
+            Self::fault_injection_pause_nakamoto_block_processing();
+
             // process at most one block per loop pass
             let mut processed_block_receipt = match NakamotoChainState::process_next_nakamoto_block(
                 &mut self.chain_state_db,

stackslib/src/chainstate/nakamoto/miner.rs

@@ -401,11 +401,11 @@ impl NakamotoBlockBuilder {
         burn_dbconn: &'a SortitionHandleConn,
         info: &'b mut MinerTenureInfo<'a>,
     ) -> Result<ClarityTx<'b, 'b>, Error> {
-        if info.tenure_block_commit_opt.is_none() {
+        let Some(block_commit) = info.tenure_block_commit_opt.as_ref() else {
             return Err(Error::InvalidStacksBlock(
                 "Block-commit is required; cannot mine a shadow block".into(),
             ));
-        }
+        };
 
         let SetupBlockResult {
             clarity_tx,
@@ -426,10 +426,7 @@ impl NakamotoBlockBuilder {
             info.coinbase_height,
             info.cause == Some(TenureChangeCause::Extended),
             &self.header.pox_treatment,
-            // safety: checked above
-            info.tenure_block_commit_opt
-                .as_ref()
-                .unwrap_or_else(|| panic!("FATAL: no block-commit for normal Nakamoto block")),
+            block_commit,
             &info.active_reward_set,
         )?;
         self.matured_miner_rewards_opt = matured_miner_rewards_opt;

stackslib/src/chainstate/nakamoto/shadow.rs

@@ -55,6 +55,7 @@ use crate::chainstate::nakamoto::{
     SortitionHandleConn, StacksDBIndexed,
 };
 use crate::chainstate::stacks::boot::RewardSet;
+use crate::chainstate::stacks::db::blocks::DummyEventDispatcher;
 use crate::chainstate::stacks::db::{
     ChainstateTx, ClarityTx, StacksAccount, StacksChainState, StacksHeaderInfo,
 };
@@ -70,6 +71,7 @@ use crate::chainstate::stacks::{
 use crate::clarity::vm::types::StacksAddressExtensions;
 use crate::clarity_vm::clarity::ClarityInstance;
 use crate::clarity_vm::database::SortitionDBRef;
+use crate::net::Error as NetError;
 use crate::util_lib::db::{query_row, u64_to_sql, Error as DBError};
 
 impl NakamotoBlockHeader {
@@ -461,7 +463,7 @@ impl NakamotoBlockBuilder {
     }
 
     /// Get an address's account
-    fn get_account(
+    pub fn get_account(
         chainstate: &mut StacksChainState,
         sortdb: &SortitionDB,
         addr: &StacksAddress,
@@ -561,13 +563,17 @@ impl NakamotoBlockBuilder {
         }
         let block = builder.mine_nakamoto_block(&mut tenure_tx);
         let size = builder.bytes_so_far;
-        let cost = builder.tenure_finish(tenure_tx).unwrap();
+        let cost = builder.tenure_finish(tenure_tx)?;
         Ok((block, size, cost))
     }
 
     /// Produce a single-block shadow tenure.
     /// Used by tooling to synthesize shadow blocks in case of an emergency.
-    /// The details and circumstances will be recorded in an accompanying SIP.
+    /// The details and circumatances will be recorded in an accompanying SIP.
+    ///
+    /// `naka_tip_id` is the Stacks chain tip on top of which the shadow block will be built.
+    /// `tenure_id_consensus_hash` is the sortition in which the shadow block will be built.
+    /// `txs` are transactions to include, beyond a coinbase and tenure-change
     pub fn make_shadow_tenure(
         chainstate: &mut StacksChainState,
         sortdb: &SortitionDB,
@@ -704,8 +710,7 @@ impl NakamotoBlockBuilder {
             Some(&tenure_change_tx),
             Some(&coinbase_tx),
             1,
-        )
-        .unwrap();
+        )?;
 
         let mut block_txs = vec![tenure_change_tx, coinbase_tx];
         block_txs.append(&mut txs);
@@ -852,3 +857,134 @@ impl<'a> NakamotoStagingBlocksTx<'a> {
         Ok(())
     }
 }
+
+/// DO NOT RUN ON A RUNNING NODE (unless you're testing).
+///
+/// Insert and process a shadow block into the Stacks chainstate.
+pub fn process_shadow_block(
+    chain_state: &mut StacksChainState,
+    sort_db: &mut SortitionDB,
+    shadow_block: NakamotoBlock,
+) -> Result<(), ChainstateError> {
+    let tx = chain_state.staging_db_tx_begin()?;
+    tx.add_shadow_block(&shadow_block)?;
+    tx.commit()?;
+
+    let no_dispatch: Option<DummyEventDispatcher> = None;
+    loop {
+        let sort_tip = SortitionDB::get_canonical_burn_chain_tip(sort_db.conn())?;
+
+        // process at most one block per loop pass
+        let processed_block_receipt = match NakamotoChainState::process_next_nakamoto_block(
+            chain_state,
+            sort_db,
+            &sort_tip.sortition_id,
+            no_dispatch.as_ref(),
+        ) {
+            Ok(receipt_opt) => receipt_opt,
+            Err(ChainstateError::InvalidStacksBlock(msg)) => {
+                warn!("Encountered invalid block: {}", &msg);
+                continue;
+            }
+            Err(ChainstateError::NetError(NetError::DeserializeError(msg))) => {
+                // happens if we load a zero-sized block (i.e. an invalid block)
+                warn!("Encountered invalid block (codec error): {}", &msg);
+                continue;
+            }
+            Err(e) => {
+                // something else happened
+                return Err(e.into());
+            }
+        };
+
+        if processed_block_receipt.is_none() {
+            // out of blocks
+            info!("No more blocks to process (no receipts)");
+            break;
+        };
+
+        let Some((_, processed, orphaned, _)) = chain_state
+            .nakamoto_blocks_db()
+            .get_block_processed_and_signed_weight(
+                &shadow_block.header.consensus_hash,
+                &shadow_block.header.block_hash(),
+            )?
+        else {
+            return Err(ChainstateError::InvalidStacksBlock(format!(
+                "Shadow block {} for tenure {} not store",
+                &shadow_block.block_id(),
+                &shadow_block.header.consensus_hash
+            )));
+        };
+
+        if orphaned {
+            return Err(ChainstateError::InvalidStacksBlock(format!(
+                "Shadow block {} for tenure {} was orphaned",
+                &shadow_block.block_id(),
+                &shadow_block.header.consensus_hash
+            )));
+        }
+
+        if processed {
+            break;
+        }
+    }
+    Ok(())
+}
+
+/// DO NOT RUN ON A RUNNING NODE (unless you're testing).
+///
+/// Automatically repair a node that has been stalled due to an empty prepare phase.
+/// Works by synthesizing, inserting, and processing shadow tenures in-between the last sortition
+/// with a winner and the burnchain tip.
+///
+/// This is meant to be accessed by the tooling. Once the blocks are synthesized, they would be
+/// added into other broken nodes' chainstates by the same tooling.  Ultimately, a patched node
+/// would be released with these shadow blocks added in as part of the chainstate schema.
+///
+/// Returns the syntheisized shadow blocks on success.
+/// Returns error on failure.
+pub fn shadow_chainstate_repair(
+    chain_state: &mut StacksChainState,
+    sort_db: &mut SortitionDB,
+) -> Result<Vec<NakamotoBlock>, ChainstateError> {
+    let sort_tip = SortitionDB::get_canonical_burn_chain_tip(sort_db.conn())?;
+
+    let header = NakamotoChainState::get_canonical_block_header(chain_state.db(), &sort_db)?
+        .ok_or_else(|| ChainstateError::NoSuchBlockError)?;
+
+    let header_sn =
+        SortitionDB::get_block_snapshot_consensus(sort_db.conn(), &header.consensus_hash)?
+            .ok_or_else(|| {
+                ChainstateError::InvalidStacksBlock(
+                    "Canonical stacks header does not have a sortition".into(),
+                )
+            })?;
+
+    let mut shadow_blocks = vec![];
+    for burn_height in (header_sn.block_height + 1)..sort_tip.block_height {
+        let sort_tip = SortitionDB::get_canonical_burn_chain_tip(sort_db.conn())?;
+        let sort_handle = sort_db.index_handle(&sort_tip.sortition_id);
+        let sn = sort_handle
+            .get_block_snapshot_by_height(burn_height)?
+            .ok_or_else(|| ChainstateError::InvalidStacksBlock("No sortition at height".into()))?;
+
+        let header = NakamotoChainState::get_canonical_block_header(chain_state.db(), &sort_db)?
+            .ok_or_else(|| ChainstateError::NoSuchBlockError)?;
+
+        let chain_tip = header.index_block_hash();
+        let shadow_block = NakamotoBlockBuilder::make_shadow_tenure(
+            chain_state,
+            sort_db,
+            chain_tip.clone(),
+            sn.consensus_hash,
+            vec![],
+        )?;
+
+        shadow_blocks.push(shadow_block.clone());
+
+        process_shadow_block(chain_state, sort_db, shadow_block)?;
+    }
+
+    Ok(shadow_blocks)
+}

stackslib/src/chainstate/nakamoto/staging_blocks.rs

@@ -278,7 +278,7 @@ impl<'a> NakamotoStagingBlocksConnRef<'a> {
     /// There will be at most one such block.
     ///
     /// NOTE: for Nakamoto blocks, the sighash is the same as the block hash.
-    pub(crate) fn get_block_processed_and_signed_weight(
+    pub fn get_block_processed_and_signed_weight(
         &self,
         consensus_hash: &ConsensusHash,
         block_hash: &BlockHeaderHash,