Unify and limit rocksdb dependency places (#8371)

* secret_store: remove kvdb_rocksdb dependency

* cli: init db mod for open dispatch

* cli: move db, client_db, restoration_db, secretstore_db to a separate mod

* migration: rename to migration-rocksdb and remove ethcore-migrations

* ethcore: re-move kvdb-rocksdb dep to test

* mark test_helpers as test only and fix migration mod naming

* Move restoration_db_handler to test_helpers_internal

* Fix missing preambles in test_helpers_internal and rocksdb/helpers

* Move test crates downward

* Fix missing docs

* cli, db::open_db: move each argument to a separate line

* Use featuregate instead of dead code for `open_secretstore_db`

* Move pathbuf import to open_secretstore_db

Because it's only used there behind a feature gate

Author: sorpaas

Cargo.lock

@@ -37,7 +37,6 @@ ethcore-bytes = { path = "util/bytes" }
 ethcore-io = { path = "util/io" }
 ethcore-light = { path = "ethcore/light" }
 ethcore-logger = { path = "logger" }
-ethcore-migrations = { path = "ethcore/migrations" }
 ethcore-miner = { path = "miner" }
 ethcore-network = { path = "util/network" }
 ethcore-private-tx = { path = "ethcore/private-tx" }
@@ -64,7 +63,7 @@ path = { path = "util/path" }
 dir = { path = "util/dir" }
 panic_hook = { path = "util/panic_hook" }
 keccak-hash = { path = "util/hash" }
-migration = { path = "util/migration" }
+migration-rocksdb = { path = "util/migration-rocksdb" }
 kvdb = { path = "util/kvdb" }
 kvdb-rocksdb = { path = "util/kvdb-rocksdb" }
 journaldb = { path = "util/journaldb" }

Cargo.toml

@@ -51,7 +51,6 @@ rlp_compress = { path = "../util/rlp_compress" }
 rlp_derive = { path = "../util/rlp_derive" }
 kvdb = { path = "../util/kvdb" }
 kvdb-memorydb = { path = "../util/kvdb-memorydb" }
-kvdb-rocksdb = { path = "../util/kvdb-rocksdb" }
 util-error = { path = "../util/error" }
 snappy = { git = "https://github.com/paritytech/rust-snappy" }
 stop-guard = { path = "../util/stop-guard" }
@@ -71,6 +70,7 @@ journaldb = { path = "../util/journaldb" }
 [dev-dependencies]
 tempdir = "0.3"
 trie-standardmap = { path = "../util/trie-standardmap" }
+kvdb-rocksdb = { path = "../util/kvdb-rocksdb" }
 
 [features]
 # Display EVM debug traces.

ethcore/Cargo.toml

@@ -92,7 +92,6 @@ extern crate ansi_term;
 extern crate unexpected;
 extern crate kvdb;
 extern crate kvdb_memorydb;
-extern crate kvdb_rocksdb;
 extern crate util_error;
 extern crate snappy;
 
@@ -128,6 +127,9 @@ extern crate evm;
 
 pub extern crate ethstore;
 
+#[cfg(test)]
+extern crate kvdb_rocksdb;
+
 pub mod account_provider;
 pub mod block;
 pub mod client;
@@ -167,6 +169,8 @@ mod tests;
 #[cfg(test)]
 #[cfg(feature="json-tests")]
 mod json_tests;
+#[cfg(test)]
+mod test_helpers_internal;
 
 pub use types::*;
 pub use executive::contract_address;

ethcore/migrations/Cargo.toml

@@ -635,7 +635,7 @@ mod tests {
 	use snapshot::{ManifestData, RestorationStatus, SnapshotService};
 	use super::*;
 	use tempdir::TempDir;
-	use test_helpers::restoration_db_handler;
+	use test_helpers_internal::restoration_db_handler;
 
 	struct NoopDBRestore;
 	impl DatabaseRestore for NoopDBRestore {

ethcore/src/lib.rs

@@ -24,7 +24,8 @@ use ids::BlockId;
 use snapshot::service::{Service, ServiceParams};
 use snapshot::{self, ManifestData, SnapshotService};
 use spec::Spec;
-use test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler};
+use test_helpers::generate_dummy_client_with_spec_and_data;
+use test_helpers_internal::restoration_db_handler;
 
 use io::IoChannel;
 use kvdb_rocksdb::{Database, DatabaseConfig};

ethcore/src/snapshot/service.rs

@@ -35,11 +35,8 @@ use spec::Spec;
 use state_db::StateDB;
 use state::*;
 use std::sync::Arc;
-use std::path::Path;
 use transaction::{Action, Transaction, SignedTransaction};
 use views::BlockView;
-use kvdb::{KeyValueDB, KeyValueDBHandler};
-use kvdb_rocksdb::{Database, DatabaseConfig};
 
 /// Creates test block with corresponding header
 pub fn create_test_block(header: &Header) -> Bytes {
@@ -402,20 +399,3 @@ impl ChainNotify for TestNotify {
 		self.messages.write().push(data);
 	}
 }
-
-/// Creates new instance of KeyValueDBHandler
-pub fn restoration_db_handler(config: DatabaseConfig) -> Box<KeyValueDBHandler> {
-	use kvdb::Error;
-
-	struct RestorationDBHandler {
-		config: DatabaseConfig,
-	}
-
-	impl KeyValueDBHandler for RestorationDBHandler {
-		fn open(&self, db_path: &Path) -> Result<Arc<KeyValueDB>, Error> {
-			Ok(Arc::new(Database::open(&self.config, &db_path.to_string_lossy())?))
-		}
-	}
-
-	Box::new(RestorationDBHandler { config })
-}

ethcore/src/snapshot/tests/service.rs

@@ -0,0 +1,39 @@
+// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Internal helpers for client tests
+
+use std::path::Path;
+use std::sync::Arc;
+use kvdb::{KeyValueDB, KeyValueDBHandler};
+use kvdb_rocksdb::{Database, DatabaseConfig};
+
+/// Creates new instance of KeyValueDBHandler
+pub fn restoration_db_handler(config: DatabaseConfig) -> Box<KeyValueDBHandler> {
+	use kvdb::Error;
+
+	struct RestorationDBHandler {
+		config: DatabaseConfig,
+	}
+
+	impl KeyValueDBHandler for RestorationDBHandler {
+		fn open(&self, db_path: &Path) -> Result<Arc<KeyValueDB>, Error> {
+			Ok(Arc::new(Database::open(&self.config, &db_path.to_string_lossy())?))
+		}
+	}
+
+	Box::new(RestorationDBHandler { config })
+}

ethcore/src/test_helpers.rs

@@ -27,20 +27,19 @@ use bytes::ToPretty;
 use rlp::PayloadInfo;
 use ethcore::account_provider::AccountProvider;
 use ethcore::client::{Mode, DatabaseCompactionProfile, VMType, BlockImportError, Nonce, Balance, BlockChainClient, BlockId, BlockInfo, ImportBlock};
-use ethcore::db::NUM_COLUMNS;
 use ethcore::error::ImportError;
 use ethcore::miner::Miner;
 use ethcore::verification::queue::VerifierSettings;
 use ethcore_service::ClientService;
 use cache::CacheConfig;
 use informant::{Informant, FullNodeInformantData, MillisecondDuration};
-use kvdb_rocksdb::{Database, DatabaseConfig};
 use params::{SpecType, Pruning, Switch, tracing_switch_to_bool, fatdb_switch_to_bool};
-use helpers::{to_client_config, execute_upgrades, open_client_db, client_db_config, restoration_db_handler, compaction_profile};
+use helpers::{to_client_config, execute_upgrades};
 use dir::Directories;
 use user_defaults::UserDefaults;
 use fdlimit;
 use ethcore_private_tx;
+use db;
 
 #[derive(Debug, PartialEq)]
 pub enum DataFormat {
@@ -188,8 +187,7 @@ fn execute_import_light(cmd: ImportBlockchain) -> Result<(), String> {
 	let client_path = db_dirs.client_path(algorithm);
 
 	// execute upgrades
-	let compaction = compaction_profile(&cmd.compaction, db_dirs.db_root_path().as_path());
-	execute_upgrades(&cmd.dirs.base, &db_dirs, algorithm, compaction)?;
+	execute_upgrades(&cmd.dirs.base, &db_dirs, algorithm, &cmd.compaction)?;
 
 	// create dirs used by parity
 	cmd.dirs.create_dirs(false, false, false)?;
@@ -210,19 +208,10 @@ fn execute_import_light(cmd: ImportBlockchain) -> Result<(), String> {
 	config.queue.verifier_settings = cmd.verifier_settings;
 
 	// initialize database.
-	let db = {
-		let db_config = DatabaseConfig {
-			memory_budget: Some(cmd.cache_config.blockchain() as usize * 1024 * 1024),
-			compaction: compaction,
-			wal: cmd.wal,
-			.. DatabaseConfig::with_columns(NUM_COLUMNS)
-		};
-
-		Arc::new(Database::open(
-			&db_config,
-			&client_path.to_str().expect("DB path could not be converted to string.")
-		).map_err(|e| format!("Failed to open database: {}", e))?)
-	};
+	let db = db::open_db(&client_path.to_str().expect("DB path could not be converted to string."),
+						 &cmd.cache_config,
+						 &cmd.compaction,
+						 cmd.wal)?;
 
 	// TODO: could epoch signals be avilable at the end of the file?
 	let fetch = ::light::client::fetch::unavailable();
@@ -354,7 +343,7 @@ fn execute_import(cmd: ImportBlockchain) -> Result<(), String> {
 	let snapshot_path = db_dirs.snapshot_path();
 
 	// execute upgrades
-	execute_upgrades(&cmd.dirs.base, &db_dirs, algorithm, compaction_profile(&cmd.compaction, db_dirs.db_root_path().as_path()))?;
+	execute_upgrades(&cmd.dirs.base, &db_dirs, algorithm, &cmd.compaction)?;
 
 	// create dirs used by parity
 	cmd.dirs.create_dirs(false, false, false)?;
@@ -378,9 +367,8 @@ fn execute_import(cmd: ImportBlockchain) -> Result<(), String> {
 
 	client_config.queue.verifier_settings = cmd.verifier_settings;
 
-	let client_db_config = client_db_config(&client_path, &client_config);
-	let client_db = open_client_db(&client_path, &client_db_config)?;
-	let restoration_db_handler = restoration_db_handler(client_db_config);
+	let client_db = db::open_client_db(&client_path, &client_config)?;
+	let restoration_db_handler = db::restoration_db_handler(&client_path, &client_config);
 
 	// build client
 	let service = ClientService::start(
@@ -549,7 +537,7 @@ fn start_client(
 	let snapshot_path = db_dirs.snapshot_path();
 
 	// execute upgrades
-	execute_upgrades(&dirs.base, &db_dirs, algorithm, compaction_profile(&compaction, db_dirs.db_root_path().as_path()))?;
+	execute_upgrades(&dirs.base, &db_dirs, algorithm, &compaction)?;
 
 	// create dirs used by parity
 	dirs.create_dirs(false, false, false)?;
@@ -571,9 +559,8 @@ fn start_client(
 		true,
 	);
 
-	let client_db_config = client_db_config(&client_path, &client_config);
-	let client_db = open_client_db(&client_path, &client_db_config)?;
-	let restoration_db_handler = restoration_db_handler(client_db_config);
+	let client_db = db::open_client_db(&client_path, &client_config)?;
+	let restoration_db_handler = db::restoration_db_handler(&client_path, &client_config);
 
 	let service = ClientService::start(
 		client_config,

ethcore/src/test_helpers_internal.rs

@@ -1,4 +1,4 @@
-// Copyright 2015-2017 Parity Technologies (UK) Ltd.
+// Copyright 2015-2018 Parity Technologies (UK) Ltd.
 // This file is part of Parity.
 
 // Parity is free software: you can redistribute it and/or modify
@@ -14,24 +14,12 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Database migrations.
+//! Database-related operations.
 
-extern crate migration;
+#[path="rocksdb/mod.rs"]
+mod impls;
 
-use migration::ChangeColumns;
+pub use self::impls::{open_db, open_client_db, restoration_db_handler, migrate};
 
-/// The migration from v10 to v11.
-/// Adds a column for node info.
-pub const TO_V11: ChangeColumns = ChangeColumns {
-	pre_columns: Some(6),
-	post_columns: Some(7),
-	version: 11,
-};
-
-/// The migration from v11 to v12.
-/// Adds a column for light chain storage.
-pub const TO_V12: ChangeColumns = ChangeColumns {
-	pre_columns: Some(7),
-	post_columns: Some(8),
-	version: 12,
-};
+#[cfg(feature = "secretstore")]
+pub use self::impls::open_secretstore_db;

parity/blockchain.rs

@@ -0,0 +1,38 @@
+// Copyright 2015-2018 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::path::Path;
+use ethcore::db::NUM_COLUMNS;
+use ethcore::client::{ClientConfig, DatabaseCompactionProfile};
+use super::kvdb_rocksdb::{CompactionProfile, DatabaseConfig};
+
+pub fn compaction_profile(profile: &DatabaseCompactionProfile, db_path: &Path) -> CompactionProfile {
+	match profile {
+		&DatabaseCompactionProfile::Auto => CompactionProfile::auto(db_path),
+		&DatabaseCompactionProfile::SSD => CompactionProfile::ssd(),
+		&DatabaseCompactionProfile::HDD => CompactionProfile::hdd(),
+	}
+}
+
+pub fn client_db_config(client_path: &Path, client_config: &ClientConfig) -> DatabaseConfig {
+	let mut client_db_config = DatabaseConfig::with_columns(NUM_COLUMNS);
+
+	client_db_config.memory_budget = client_config.db_cache_size;
+	client_db_config.compaction = compaction_profile(&client_config.db_compaction, &client_path);
+	client_db_config.wal = client_config.db_wal;
+
+	client_db_config
+}