Pruning logic (#4194)

* add parallel code

* parallel execution

* create chain when the parents are ready in parallel execution

* set executing state in sender

* add 10000 buffer for parallel execution

* add log for saving time

* add some test

* add false testing case

* add more testing code

* add check data

* add verify blue block in verifier

* add some code

* add verification

* fix some bugs

* add yeilding after execution for processing the main chain in other service

* fmt and clippy

* rebase master
set the number 4500000 for vega updating

* 3300000 will be version 1 in vega

* add pruning logic and compatible logic
fix some test cases

* fix clippy

* fix test case

* remove some single chain test case

* fix clippy

* fix flexdag's test case

* fix clippy

* add rational random to pass the deserialization verification

* merge dag-master

* fix bugs: blockmeta changes into latest version

* add update db code

* add dag db update

* update dag db

* update dag db

* uncomment the pruning code

* rebase sync parallel3

* fix compiling

* db update for dag state

* fix connection

* add verify pruning

* add pruning height

* no checking the pruning point if the main header still dose not have the pruning point

* add is ancestor of command for reachability viewing

* add command file

* use 850000

* add rpc json new command

* add some log for debug

* if this is the first pruning point, the previous one will be genesis

* save the dag state using the pruning point as the key and if it is 0, use genesis id

* merge dag master

* get the tips by genesis id if the pruning point is 0

* fix test case as prievious version did

* use genesis id to get the ghost data

* add genesis in registry in test_miner_service

* use 1000 as parallel buffer

* remove some logs
add test case for pruning

* add pruning arguments in pruning methods to custom the network config

* add test case for pruning calculation and pruning

* add write lock when saving the tips

* add test pruning for chain

* add test case

* if no fork, execute at once

* add test code

* add some rpc param in yaml's cmd

* use macro to define the parallel count

* use 4090000 as pruning beginning

* fix the code for robot's comments

chain/api/src/chain.rs

@@ -106,14 +106,16 @@ pub trait ChainReader {
         access_path: Option<AccessPath>,
     ) -> Result<Option<TransactionInfoWithProof>>;
 
-    fn current_tips_hash(&self) -> Result<Vec<HashValue>>;
+    fn current_tips_hash(&self, pruning_point: HashValue) -> Result<Vec<HashValue>>;
     fn has_dag_block(&self, header_id: HashValue) -> Result<bool>;
     fn check_chain_type(&self) -> Result<ChainType>;
     fn verify_and_ghostdata(
         &self,
         uncles: &[BlockHeader],
         header: &BlockHeader,
     ) -> Result<GhostdagData>;
+    fn is_dag_ancestor_of(&self, ancestor: HashValue, descendants: Vec<HashValue>) -> Result<bool>;
+    fn get_pruning_height(&self) -> BlockNumber;
 }
 
 pub trait ChainWriter {

chain/api/src/message.rs

@@ -4,7 +4,7 @@
 use crate::{ChainType, TransactionInfoWithProof};
 use anyhow::Result;
 use starcoin_crypto::HashValue;
-use starcoin_dag::consensusdb::consenses_state::DagStateView;
+use starcoin_dag::consensusdb::consenses_state::{DagStateView, ReachabilityView};
 use starcoin_dag::types::ghostdata::GhostdagData;
 use starcoin_service_registry::ServiceRequest;
 use starcoin_types::transaction::RichTransactionInfo;
@@ -68,6 +68,10 @@ pub enum ChainRequest {
     GetDagStateView,
     CheckChainType,
     GetGhostdagData(HashValue),
+    IsAncestorOfCommand {
+        ancestor: HashValue,
+        descendants: Vec<HashValue>,
+    },
 }
 
 impl ServiceRequest for ChainRequest {
@@ -99,4 +103,5 @@ pub enum ChainResponse {
     DagStateView(Box<DagStateView>),
     CheckChainType(ChainType),
     GhostdagDataOption(Box<Option<GhostdagData>>),
+    IsAncestorOfCommand { reachability_view: ReachabilityView },
 }

chain/api/src/service.rs

@@ -5,7 +5,7 @@ use crate::message::{ChainRequest, ChainResponse};
 use crate::{ChainType, TransactionInfoWithProof};
 use anyhow::{bail, Result};
 use starcoin_crypto::HashValue;
-use starcoin_dag::consensusdb::consenses_state::DagStateView;
+use starcoin_dag::consensusdb::consenses_state::{DagStateView, ReachabilityView};
 use starcoin_dag::types::ghostdata::GhostdagData;
 use starcoin_service_registry::{ActorService, ServiceHandler, ServiceRef};
 use starcoin_types::contract_event::{ContractEvent, ContractEventInfo};
@@ -149,6 +149,11 @@ pub trait ChainAsyncService:
     async fn get_dag_state(&self) -> Result<DagStateView>;
     async fn check_chain_type(&self) -> Result<ChainType>;
     async fn get_ghostdagdata(&self, id: HashValue) -> Result<Option<GhostdagData>>;
+    async fn is_ancestor_of(
+        &self,
+        ancestor: HashValue,
+        descendants: Vec<HashValue>,
+    ) -> Result<starcoin_dag::consensusdb::consenses_state::ReachabilityView>;
 }
 
 #[async_trait::async_trait]
@@ -486,4 +491,21 @@ where
             bail!("failed to get ghostdag data")
         }
     }
+    async fn is_ancestor_of(
+        &self,
+        ancestor: HashValue,
+        descendants: Vec<HashValue>,
+    ) -> Result<ReachabilityView> {
+        let response = self
+            .send(ChainRequest::IsAncestorOfCommand {
+                ancestor,
+                descendants,
+            })
+            .await??;
+        if let ChainResponse::IsAncestorOfCommand { reachability_view } = response {
+            Ok(reachability_view)
+        } else {
+            bail!("failed to get ghostdag data")
+        }
+    }
 }

chain/mock/src/mock_chain.rs

@@ -1,17 +1,18 @@
 // Copyright (c) The Starcoin Core Contributors
 // SPDX-License-Identifier: Apache-2.0
 
-use anyhow::Result;
+use anyhow::{format_err, Result};
 use starcoin_account_api::AccountInfo;
 use starcoin_chain::{BlockChain, ChainReader, ChainWriter};
 use starcoin_config::ChainNetwork;
 use starcoin_consensus::Consensus;
 use starcoin_crypto::HashValue;
-use starcoin_dag::blockdag::BlockDAG;
+use starcoin_dag::blockdag::{BlockDAG, MineNewDagBlockInfo};
 use starcoin_genesis::Genesis;
 use starcoin_logger::prelude::*;
 use starcoin_storage::Storage;
 use starcoin_types::block::{Block, BlockHeader};
+use starcoin_types::blockhash::KType;
 use starcoin_types::startup_info::ChainInfo;
 use std::sync::Arc;
 use std::vec;
@@ -39,6 +40,26 @@ impl MockChain {
         Ok(Self::new_inner(net, chain, miner, storage))
     }
 
+    pub fn new_with_params(
+        net: ChainNetwork,
+        k: KType,
+        pruning_depth: u64,
+        pruning_finality: u64,
+    ) -> Result<Self> {
+        let (storage, chain_info, _, dag) =
+            Genesis::init_storage_for_test_with_param(&net, k, pruning_depth, pruning_finality)?;
+
+        let chain = BlockChain::new(
+            net.time_service(),
+            chain_info.head().id(),
+            storage.clone(),
+            None,
+            dag,
+        )?;
+        let miner = AccountInfo::random();
+        Ok(Self::new_inner(net, chain, miner, storage))
+    }
+
     pub fn new_with_storage(
         net: ChainNetwork,
         storage: Arc<Storage>,
@@ -178,6 +199,26 @@ impl MockChain {
             .create_block(template, self.net.time_service().as_ref())
     }
 
+    pub fn produce_block_by_params(
+        &mut self,
+        parent_header: BlockHeader,
+        tips: Vec<HashValue>,
+        pruning_point: HashValue,
+    ) -> Result<Block> {
+        let (template, _) = self.head.create_block_template_by_header(
+            *self.miner.address(),
+            parent_header,
+            vec![],
+            vec![],
+            None,
+            tips,
+            pruning_point,
+        )?;
+        self.head
+            .consensus()
+            .create_block(template, self.net.time_service().as_ref())
+    }
+
     pub fn produce_block_by_tips(
         &mut self,
         parent_header: BlockHeader,
@@ -197,6 +238,72 @@ impl MockChain {
             .create_block(template, self.net.time_service().as_ref())
     }
 
+    pub fn produce_block_for_pruning(&mut self) -> Result<Block> {
+        let tips = self.head.get_dag_state()?.tips;
+        let ghostdata = self.head.dag().ghost_dag_manager().ghostdag(&tips)?;
+        let selected_header = self
+            .head()
+            .get_storage()
+            .get_block_header_by_hash(ghostdata.selected_parent)?
+            .ok_or_else(|| {
+                format_err!(
+                    "Cannot find block header by hash: {:?}",
+                    ghostdata.selected_parent
+                )
+            })?;
+
+        let previous_pruning = if selected_header.pruning_point() == HashValue::zero() {
+            self.head().get_storage().get_genesis()?.unwrap()
+        } else {
+            selected_header.pruning_point()
+        };
+
+        let prevous_ghostdata = self
+            .head()
+            .dag()
+            .ghostdata_by_hash(previous_pruning)?
+            .ok_or_else(|| format_err!("Cannot find ghostdata by hash: {:?}", previous_pruning))?;
+
+        let MineNewDagBlockInfo {
+            tips: pruned_tips,
+            blue_blocks,
+            pruning_point,
+        } = self
+            .head
+            .dag()
+            .calc_mergeset_and_tips(previous_pruning, prevous_ghostdata.as_ref())?;
+
+        debug!(
+            "tips: {:?}, blue_blocks: {:?}, pruning_point: {:?}",
+            pruned_tips, blue_blocks, pruning_point
+        );
+
+        let (template, _) = self.head.create_block_template_by_header(
+            *self.miner.address(),
+            selected_header,
+            vec![],
+            blue_blocks
+                .get(1..)
+                .unwrap_or(&[])
+                .iter()
+                .map(|block_id| {
+                    self.head()
+                        .get_storage()
+                        .get_block_header_by_hash(*block_id)?
+                        .ok_or_else(|| {
+                            format_err!("Block header not found for hash: {:?}", block_id)
+                        })
+                })
+                .collect::<Result<Vec<_>>>()?,
+            None,
+            pruned_tips,
+            pruning_point,
+        )?;
+        self.head
+            .consensus()
+            .create_block(template, self.net.time_service().as_ref())
+    }
+
     pub fn apply(&mut self, block: Block) -> Result<()> {
         self.head.apply(block)?;
         Ok(())

chain/service/src/chain_service.rs

@@ -254,6 +254,12 @@ impl ServiceHandler<Self, ChainRequest> for ChainReaderService {
             ChainRequest::GetGhostdagData(id) => Ok(ChainResponse::GhostdagDataOption(Box::new(
                 self.inner.get_ghostdagdata(id)?,
             ))),
+            ChainRequest::IsAncestorOfCommand {
+                ancestor,
+                descendants,
+            } => Ok(ChainResponse::IsAncestorOfCommand {
+                reachability_view: self.inner.dag.is_ancestor_of(ancestor, descendants)?,
+            }),
         }
     }
 }