Skip to content

Commit

Permalink
[storage] Introduce rooch-db crate and refactor the StoreConfig (#1819)
Browse files Browse the repository at this point in the history 
* [storage] Introduce rooch-db crate and refactor the StoreConfig

* update scripts for clean command
  • Loading branch information
@jolestar
jolestar authored Jun 5, 2024
1 parent dc5bbe7 commit 9912342
Show file tree
Hide file tree
Showing 36 changed files with 428 additions and 759 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/docker_build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,5 +85,5 @@ jobs:
chmod 600 private_key.pem
sudo apt update
sudo apt install -y --no-install-recommends openssh-server
ssh -o StrictHostKeyChecking=no -i private_key.pem $USER@$HOST bash -c "'sleep 30' && docker image prune -a -f && docker ps | grep main_debug | awk '{print \$1}' | xargs -r docker stop && docker ps -a | grep main_debug | awk '{print \$1}' | xargs -r docker rm -f && docker pull 'ghcr.io/rooch-network/rooch:main_debug' && docker run --rm -v /root:/root ghcr.io/rooch-network/rooch:main_debug server clean -n dev && docker run -d -v /root:/root -p 50051:50051 'ghcr.io/rooch-network/rooch:main_debug' server start -n dev --btc-rpc-url '${{secrets.BTC_REGTEST_RPC_URL}}' --btc-rpc-username rooch-regtest --btc-rpc-password '${{secrets.BTC_REGTEST_RPC_PWD}}' --da '{\"internal-da-server\": {\"servers\": [{\"open-da\": {\"scheme\": \"fs\"}}]}}'"
ssh -o StrictHostKeyChecking=no -i private_key.pem $USER@$HOST bash -c "'sleep 30' && docker image prune -a -f && docker ps | grep main_debug | awk '{print \$1}' | xargs -r docker stop && docker ps -a | grep main_debug | awk '{print \$1}' | xargs -r docker rm -f && docker pull 'ghcr.io/rooch-network/rooch:main_debug' && docker run --rm -v /root:/root ghcr.io/rooch-network/rooch:main_debug server clean -n dev -f && docker run -d -v /root:/root -p 50051:50051 'ghcr.io/rooch-network/rooch:main_debug' server start -n dev --btc-rpc-url '${{secrets.BTC_REGTEST_RPC_URL}}' --btc-rpc-username rooch-regtest --btc-rpc-password '${{secrets.BTC_REGTEST_RPC_PWD}}' --da '{\"internal-da-server\": {\"servers\": [{\"open-da\": {\"scheme\": \"fs\"}}]}}'"
ssh -o StrictHostKeyChecking=no -i private_key.pem $USER@$HOST "cd /root/rooch && git pull origin main && bash scripts/check_dev_deploy_status.sh main_debug '${{ secrets.DEV_MNEMONIC_PHRASE }}'"
32 changes: 32 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ members = [
"crates/rooch-indexer",
"crates/rooch-benchmarks",
"crates/rooch-test-transaction-builder",
"crates/rooch-db",
"crates/data_verify",
"crates/rooch-faucet",
"frameworks/framework-builder",
Expand Down Expand Up @@ -115,6 +116,7 @@ rooch-benchmarks = { path = "crates/rooch-benchmarks" }
rooch-faucet = { path = "crates/rooch-faucet" }
rooch-test-transaction-builder = { path = "crates/rooch-test-transaction-builder" }
data-verify = { path = "crates/data_verify" }
rooch-db = { path = "crates/rooch-db" }

# frameworks
framework-types = { path = "frameworks/framework-types" }
Expand Down
185 changes: 1 addition & 184 deletions crates/rooch-benchmarks/src/tx.rs
View file
Original file line number Diff line number Diff line change
@@ -1,58 +1,22 @@
// Copyright (c) RoochNetwork
// SPDX-License-Identifier: Apache-2.0

use std::fs;
use std::time::Duration;

use anyhow::Result;
use bitcoin::consensus::deserialize;
use bitcoin::hashes::Hash;
use bitcoin::hex::FromHex;
use bitcoincore_rpc_json::bitcoin;
use bitcoincore_rpc_json::bitcoin::Block;
use coerce::actor::scheduler::timer::Timer;
use coerce::actor::system::ActorSystem;
use coerce::actor::IntoActor;
use tracing::info;

use moveos_config::store_config::RocksdbConfig;
use moveos_config::DataDirPath;
use moveos_store::{MoveOSDB, MoveOSStore};
use raw_store::rocks::RocksDB;
use raw_store::StoreInstance;
use rooch_config::da_config::DAConfig;
use rooch_config::indexer_config::IndexerConfig;
use rooch_config::store_config::StoreConfig;
use rooch_da::actor::da::DAActor;
use rooch_da::proxy::DAProxy;
use rooch_executor::actor::executor::ExecutorActor;
use rooch_executor::actor::reader_executor::ReaderExecutorActor;
use rooch_executor::proxy::ExecutorProxy;
use rooch_genesis::RoochGenesis;
use rooch_indexer::actor::indexer::IndexerActor;
use rooch_indexer::actor::reader_indexer::IndexerReaderActor;
use rooch_indexer::indexer_reader::IndexerReader;
use rooch_indexer::proxy::IndexerProxy;
use rooch_indexer::IndexerStore;
use rooch_key::keystore::account_keystore::AccountKeystore;
use rooch_key::keystore::memory_keystore::InMemKeystore;
use rooch_pipeline_processor::{
actor::processor::PipelineProcessorActor, proxy::PipelineProcessorProxy,
};
use rooch_proposer::actor::messages::ProposeBlock;
use rooch_proposer::actor::proposer::ProposerActor;
use rooch_proposer::proxy::ProposerProxy;
use rooch_rpc_server::service::aggregate_service::AggregateService;
use rooch_rpc_server::service::rpc_service::RpcService;
use rooch_sequencer::actor::sequencer::SequencerActor;
use rooch_sequencer::proxy::SequencerProxy;
use rooch_store::RoochStore;
use rooch_test_transaction_builder::TestTransactionBuilder;
use rooch_types::crypto::RoochKeyPair;
use rooch_types::multichain_id::RoochMultiChainID;
use rooch_types::rooch_network::{BuiltinChainID, RoochNetwork};
use rooch_types::transaction::rooch::RoochTransaction;
use rooch_types::transaction::L1BlockWithBody;
use std::fs;

use crate::config::TxType;
use crate::tx::TxType::{Empty, Transfer};
Expand All @@ -64,153 +28,6 @@ pub fn gen_sequencer(keypair: RoochKeyPair, rooch_store: RoochStore) -> Result<S
SequencerActor::new(keypair, rooch_store.clone())
}

//TODO reuse the rpc run_start_server function
pub async fn setup_service(
datadir: &DataDirPath,
keystore: &InMemKeystore,
) -> Result<(RpcService, AggregateService)> {
// We may call `start_server` multiple times in testing scenarios
// tracing_subscriber can only be inited once.
let _ = tracing_subscriber::fmt::try_init();

let actor_system = ActorSystem::global_system();

// init storage
let (mut moveos_store, mut rooch_store) = init_storage(datadir)?;
let (mut indexer_store, indexer_reader) = init_indexer(datadir)?;

// init keystore
let rooch_account = keystore.addresses()[0];
let rooch_key_pair = keystore
.get_key_pair(&rooch_account, None)
.expect("Key pair should have value");

let sequencer_keypair = rooch_key_pair.copy();
let proposer_keypair = rooch_key_pair.copy();
let sequencer_account = sequencer_keypair.public().rooch_address()?;
let proposer_account = proposer_keypair.public().rooch_address()?;

// Init executor
let mut network: RoochNetwork = BuiltinChainID::Dev.into();
network.set_sequencer_account(sequencer_keypair.public().bitcoin_address()?);
let genesis: RoochGenesis = RoochGenesis::build(network)?;
let root = genesis.init_genesis(&mut moveos_store, &mut rooch_store, &mut indexer_store)?;

let executor_actor =
ExecutorActor::new(root.clone(), moveos_store.clone(), rooch_store.clone())?;
let reader_executor =
ReaderExecutorActor::new(root.clone(), moveos_store.clone(), rooch_store.clone())?
.into_actor(Some("ReaderExecutor"), &actor_system)
.await?;
let executor = executor_actor
.into_actor(Some("Executor"), &actor_system)
.await?;
let executor_proxy = ExecutorProxy::new(executor.into(), reader_executor.into());

// Init sequencer
info!("RPC Server sequencer address: {:?}", sequencer_account);
let sequencer = SequencerActor::new(sequencer_keypair, rooch_store.clone())?
.into_actor(Some("Sequencer"), &actor_system)
.await?;
let sequencer_proxy = SequencerProxy::new(sequencer.into());

// Init DA
let da_config = DAConfig::default();
let da_proxy = DAProxy::new(
DAActor::new(da_config, &actor_system)
.await?
.into_actor(Some("DAProxy"), &actor_system)
.await?
.into(),
);

// Init proposer
info!("RPC Server proposer address: {:?}", proposer_account);
let proposer = ProposerActor::new(proposer_keypair, da_proxy)
.into_actor(Some("Proposer"), &actor_system)
.await?;
let proposer_proxy = ProposerProxy::new(proposer.clone().into());
//TODO load from config
let block_propose_duration_in_seconds: u64 = 5;
let mut timers = vec![];
let proposer_timer = Timer::start(
proposer,
Duration::from_secs(block_propose_duration_in_seconds),
ProposeBlock {},
);
timers.push(proposer_timer);

// Init indexer
let indexer_executor = IndexerActor::new(root, indexer_store.clone(), moveos_store.clone())?
.into_actor(Some("Indexer"), &actor_system)
.await?;
let indexer_reader_executor = IndexerReaderActor::new(indexer_reader)?
.into_actor(Some("IndexerReader"), &actor_system)
.await?;
let indexer_proxy = IndexerProxy::new(indexer_executor.into(), indexer_reader_executor.into());

let processor = PipelineProcessorActor::new(
executor_proxy.clone(),
sequencer_proxy.clone(),
proposer_proxy.clone(),
indexer_proxy.clone(),
true,
)
.into_actor(Some("PipelineProcessor"), &actor_system)
.await?;
let processor_proxy = PipelineProcessorProxy::new(processor.into());

let rpc_service = RpcService::new(
executor_proxy.clone(),
sequencer_proxy,
indexer_proxy,
processor_proxy,
);
let aggregate_service = AggregateService::new(rpc_service.clone());

Ok((rpc_service, aggregate_service))
}

pub fn init_storage(datadir: &DataDirPath) -> Result<(MoveOSStore, RoochStore)> {
let (rooch_db_path, moveos_db_path) = (
StoreConfig::get_mock_rooch_store_dir(datadir),
StoreConfig::get_mock_moveos_store_dir(datadir),
);
if !rooch_db_path.exists() {
fs::create_dir_all(rooch_db_path.clone())?;
}
if !moveos_db_path.exists() {
fs::create_dir_all(moveos_db_path.clone())?;
}

//Init store
let moveosdb = MoveOSDB::new(StoreInstance::new_db_instance(RocksDB::new(
moveos_db_path,
moveos_store::StoreMeta::get_column_family_names().to_vec(),
RocksdbConfig::default(),
None,
)?))?;
let moveos_store = MoveOSStore::new(moveosdb)?;

let rooch_store = RoochStore::new(StoreInstance::new_db_instance(RocksDB::new(
rooch_db_path,
rooch_store::StoreMeta::get_column_family_names().to_vec(),
RocksdbConfig::default(),
None,
)?))?;

Ok((moveos_store, rooch_store))
}

pub fn init_indexer(datadir: &DataDirPath) -> Result<(IndexerStore, IndexerReader)> {
let indexer_db_path = IndexerConfig::get_mock_indexer_db(datadir);
let indexer_store = IndexerStore::new(indexer_db_path.clone())?;
indexer_store.create_all_tables_if_not_exists()?;
let indexer_reader = IndexerReader::new(indexer_db_path)?;

Ok((indexer_store, indexer_reader))
}

pub fn create_publish_transaction(
test_transaction_builder: &TestTransactionBuilder,
keystore: &InMemKeystore,
Expand Down
42 changes: 19 additions & 23 deletions crates/rooch-config/src/da_config.rs
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,19 @@
// Copyright (c) RoochNetwork
// SPDX-License-Identifier: Apache-2.0

use crate::config::Config;
use crate::config::{parse_hashmap, retrieve_map_config_value, MapConfigValueSource};
use crate::BaseConfig;
use clap::Parser;
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;
use std::fmt::Display;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::Arc;

use clap::Parser;
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
use serde_json::Value;

use crate::config::{parse_hashmap, retrieve_map_config_value, MapConfigValueSource};

use crate::config::Config;
use crate::{BaseConfig, RoochOpt};

static R_DEFAULT_OPENDA_FS_DIR: Lazy<PathBuf> = Lazy::new(|| PathBuf::from("openda_fs"));

#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
Expand Down Expand Up @@ -71,6 +68,17 @@ pub struct DAConfig {
// TODO external da server config
}

impl Display for DAConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}",
serde_json::to_string(self).map_err(|_| std::fmt::Error)?
)?;
Ok(())
}
}

impl Config for DAConfig {}

impl FromStr for DAConfig {
Expand All @@ -83,16 +91,7 @@ impl FromStr for DAConfig {
}

impl DAConfig {
pub fn merge_with_opt_with_init(
&mut self,
opt: &RoochOpt,
base: Arc<BaseConfig>,
with_init: bool,
) -> anyhow::Result<()> {
if let Some(ref da_config) = opt.da {
// TODO merge with field checking
*self = da_config.clone();
}
pub(crate) fn init(&mut self, base: Arc<BaseConfig>) -> anyhow::Result<()> {
self.base = Some(base);

let default_fs_root = self.get_openda_fs_dir();
Expand All @@ -108,9 +107,6 @@ impl DAConfig {
default_fs_root.to_str().unwrap(),
);
if let MapConfigValueSource::Default = var_source {
if !with_init {
continue;
}
if !default_fs_root.exists() {
std::fs::create_dir_all(default_fs_root.clone())?;
}
Expand Down
Loading

0 comments on commit 9912342

Please sign in to comment.