Make substrate generic (#169)

* Some initial work on RPC and client

* Rephrase as params

* More work on traitifying substrate.

* Traitify in_mem.rs

* traitify client.rs

* Make new primitives (mainly traits) build again.

* Many (superficial) build fixes throughout.

* Fix remaining build issues up to bft interface.

* Make bft primitives be generic.

* Switch out MisBehaviorReport for generic version.

* Merge Hashing into Header.

* Update runtime for new generics (with Hashing).

* Update demo runtime.

* Make runtime compile.

* Build fixes for runtime

* Remove old modules.

* port substrate-bft to use generic substrate types

* port client

* port substrate-test-runtime

* mostly port test-runtime to get compiling for std

* Ensure `AccountId` has a `Default`.

* Fix type deps.

* finish porting

* initialize test_runtime from genesis correctly

* remove commented code

* maybe unsigned signatures

* runtimes compile

* port over most of network

* serialization for generic types

* fix comment

* remove some unnecessary trait bounds

* network compiles

* tests compile for sync

* fix deserialization

* temporarily remove deserialize derives

* workarounds for serde issues for deriving deserialization

* get demo-runtime compiling on std

* port extrinsic-pool

* primitives reshuffling

* get network compiling again

* remove debugging file

* runtime tests now passing

* port client-db

* start to port over substrate-rpc

* mostly port over PolkadotApi

* test_runtime follows normal conventions

* substrate runtime tests pass

* deal with inherent extrinsics correctly in polkadot-api

* port transaction-pool

* port polkadot-consensus

* port substrate-rpc

* everything compiles

* tests compile

* fix grumbles

* test-runtime uses its own transfer type

* switch to master branch of jsonrpc

* fix network tests and some warnings

* all tests pass locally

* [ci-skip] add another comment about issue

* remove some curlies

Author: gavofyork

Cargo.lock

@@ -49,17 +49,18 @@ pub mod error;
 
 use std::sync::Arc;
 use client::genesis;
-use codec::Slicable;
+use demo_primitives::Hash;
 use demo_runtime::{GenesisConfig, ConsensusConfig, CouncilConfig, DemocracyConfig,
 	SessionConfig, StakingConfig, BuildExternalities};
+use demo_runtime::{Block, Header, UncheckedExtrinsic};
 use futures::{Future, Sink, Stream};
 
 struct DummyPool;
-impl extrinsic_pool::api::ExtrinsicPool for DummyPool {
+impl extrinsic_pool::api::ExtrinsicPool<UncheckedExtrinsic, Hash> for DummyPool {
 	type Error = extrinsic_pool::txpool::Error;
 
-	fn submit(&self, _: Vec<primitives::block::Extrinsic>)
-		-> Result<Vec<primitives::block::ExtrinsicHash>, Self::Error>
+	fn submit(&self, _: Vec<UncheckedExtrinsic>)
+		-> Result<Vec<Hash>, Self::Error>
 	{
 		Err("unimplemented".into())
 	}
@@ -102,8 +103,8 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 
 	struct GenesisBuilder;
 
-	impl client::GenesisBuilder for GenesisBuilder {
-		fn build(self) -> (primitives::Header, Vec<(Vec<u8>, Vec<u8>)>) {
+	impl client::GenesisBuilder<Block> for GenesisBuilder {
+		fn build(self) -> (Header, Vec<(Vec<u8>, Vec<u8>)>) {
 			let god_key = hex!["3d866ec8a9190c8343c2fc593d21d8a6d0c5c4763aaab2349de3a6111d64d124"];
 			let genesis_config = GenesisConfig {
 				consensus: Some(ConsensusConfig {
@@ -113,15 +114,15 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 				system: None,
 		//		block_time: 5,			// 5 second block time.
 				session: Some(SessionConfig {
-					validators: vec![god_key.clone()],
+					validators: vec![god_key.clone().into()],
 					session_length: 720,	// that's 1 hour per session.
 				}),
 				staking: Some(StakingConfig {
 					current_era: 0,
 					intentions: vec![],
 					transaction_base_fee: 100,
 					transaction_byte_fee: 1,
-					balances: vec![(god_key.clone(), 1u64 << 63)].into_iter().collect(),
+					balances: vec![(god_key.clone().into(), 1u64 << 63)].into_iter().collect(),
 					validator_count: 12,
 					sessions_per_era: 24,	// 24 hours per era.
 					bonding_duration: 90,	// 90 days per bond.
@@ -149,8 +150,8 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 			};
 
 			let storage = genesis_config.build_externalities();
-			let block = genesis::construct_genesis_block(&storage);
-			(primitives::block::Header::decode(&mut block.header.encode().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
+			let block = genesis::construct_genesis_block::<Block>(&storage);
+			(block.header, storage.into_iter().collect())
 		}
 	}
 
@@ -160,7 +161,7 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 	let _rpc_servers = {
 		let handler = || {
 			let chain = rpc::apis::chain::Chain::new(client.clone(), core.remote());
-			rpc::rpc_handler(client.clone(), chain, Arc::new(DummyPool), DummySystem)
+			rpc::rpc_handler::<Block, _, _, _, _>(client.clone(), chain, Arc::new(DummyPool), DummySystem)
 		};
 		let http_address = "127.0.0.1:9933".parse().unwrap();
 		let ws_address = "127.0.0.1:9944".parse().unwrap();

demo/cli/src/lib.rs

@@ -42,7 +42,7 @@ mod tests {
 	use super::Executor;
 	use substrate_executor::WasmExecutor;
 	use codec::{Slicable, Joiner};
-	use keyring::Keyring::{self, Alice, Bob};
+	use keyring::Keyring;
 	use runtime_support::{Hashable, StorageValue, StorageMap};
 	use state_machine::{CodeExecutor, TestExternalities};
 	use primitives::twox_128;
@@ -63,13 +63,21 @@ mod tests {
 		)
 	}
 
+	fn alice() -> Hash {
+		Keyring::Alice.to_raw_public().into()
+	}
+
+	fn bob() -> Hash {
+		Keyring::Bob.to_raw_public().into()
+	}
+
 	fn xt() -> UncheckedExtrinsic {
 		let extrinsic = Extrinsic {
-			signed: Alice.into(),
+			signed: alice(),
 			index: 0,
-			function: Call::Staking(staking::Call::transfer::<Concrete>(Bob.into(), 69)),
+			function: Call::Staking(staking::Call::transfer::<Concrete>(bob(), 69)),
 		};
-		let signature = Keyring::from_raw_public(extrinsic.signed).unwrap()
+		let signature = Keyring::from_raw_public(extrinsic.signed.0).unwrap()
 			.sign(&extrinsic.encode()).into();
 
 		UncheckedExtrinsic { extrinsic, signature }
@@ -82,7 +90,7 @@ mod tests {
 	#[test]
 	fn panic_execution_with_foreign_code_gives_error() {
 		let mut t: TestExternalities = map![
-			twox_128(&<staking::FreeBalance<Concrete>>::key_for(*Alice)).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0],
+			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![70u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
@@ -97,7 +105,7 @@ mod tests {
 	#[test]
 	fn panic_execution_with_native_equivalent_code_gives_error() {
 		let mut t: TestExternalities = map![
-			twox_128(&<staking::FreeBalance<Concrete>>::key_for(*Alice)).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0],
+			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![70u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
@@ -112,7 +120,7 @@ mod tests {
 	#[test]
 	fn successful_execution_with_native_equivalent_code_gives_ok() {
 		let mut t: TestExternalities = map![
-			twox_128(&<staking::FreeBalance<Concrete>>::key_for(*Alice)).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0],
+			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
@@ -124,15 +132,15 @@ mod tests {
 		assert!(r.is_ok());
 
 		runtime_io::with_externalities(&mut t, || {
-			assert_eq!(Staking::balance(&Alice), 42);
-			assert_eq!(Staking::balance(&Bob), 69);
+			assert_eq!(Staking::balance(&alice()), 42);
+			assert_eq!(Staking::balance(&bob()), 69);
 		});
 	}
 
 	#[test]
 	fn successful_execution_with_foreign_code_gives_ok() {
 		let mut t: TestExternalities = map![
-			twox_128(&<staking::FreeBalance<Concrete>>::key_for(*Alice)).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0],
+			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
@@ -144,26 +152,26 @@ mod tests {
 		assert!(r.is_ok());
 
 		runtime_io::with_externalities(&mut t, || {
-			assert_eq!(Staking::balance(&Alice), 42);
-			assert_eq!(Staking::balance(&Bob), 69);
+			assert_eq!(Staking::balance(&alice()), 42);
+			assert_eq!(Staking::balance(&bob()), 69);
 		});
 	}
 
 	fn new_test_ext() -> TestExternalities {
 		use keyring::Keyring::*;
-		let three = [3u8; 32];
+		let three = [3u8; 32].into();
 		GenesisConfig {
 			consensus: Some(Default::default()),
 			system: Some(Default::default()),
 			session: Some(SessionConfig {
 				session_length: 2,
-				validators: vec![One.into(), Two.into(), three],
+				validators: vec![One.to_raw_public().into(), Two.to_raw_public().into(), three],
 			}),
 			staking: Some(StakingConfig {
 				sessions_per_era: 2,
 				current_era: 0,
-				balances: vec![(Alice.into(), 111)],
-				intentions: vec![Alice.into(), Bob.into(), Charlie.into()],
+				balances: vec![(alice(), 111)],
+				intentions: vec![alice(), bob(), Charlie.to_raw_public().into()],
 				validator_count: 3,
 				bonding_duration: 0,
 				transaction_base_fee: 1,
@@ -178,7 +186,7 @@ mod tests {
 		use triehash::ordered_trie_root;
 
 		let extrinsics = extrinsics.into_iter().map(|extrinsic| {
-			let signature = Pair::from(Keyring::from_public(Public::from_raw(extrinsic.signed)).unwrap())
+			let signature = Pair::from(Keyring::from_public(Public::from_raw(extrinsic.signed.0)).unwrap())
 				.sign(&extrinsic.encode()).into();
 
 			UncheckedExtrinsic { extrinsic, signature }
@@ -204,9 +212,9 @@ mod tests {
 			[69u8; 32].into(),
 			hex!("76b0393b4958d3cb98bb51d9f4edb316af48485142b8721e94c3b52c75ec3243").into(),
 			vec![Extrinsic {
-				signed: Alice.into(),
+				signed: alice(),
 				index: 0,
-				function: Call::Staking(staking::Call::transfer(Bob.into(), 69)),
+				function: Call::Staking(staking::Call::transfer(bob(), 69)),
 			}]
 		)
 	}
@@ -218,14 +226,14 @@ mod tests {
 			hex!("8ae9828a5988459d35fb428086170dead660176ee0766e89bc1a4b48153d4e88").into(),
 			vec![
 				Extrinsic {
-					signed: Bob.into(),
+					signed: bob(),
 					index: 0,
-					function: Call::Staking(staking::Call::transfer(Alice.into(), 5)),
+					function: Call::Staking(staking::Call::transfer(alice(), 5)),
 				},
 				Extrinsic {
-					signed: Alice.into(),
+					signed: alice(),
 					index: 1,
-					function: Call::Staking(staking::Call::transfer(Bob.into(), 15)),
+					function: Call::Staking(staking::Call::transfer(bob(), 15)),
 				}
 			]
 		)
@@ -238,15 +246,15 @@ mod tests {
 		Executor::new().call(&mut t, COMPACT_CODE, "execute_block", &block1().0).unwrap();
 
 		runtime_io::with_externalities(&mut t, || {
-			assert_eq!(Staking::balance(&Alice), 41);
-			assert_eq!(Staking::balance(&Bob), 69);
+			assert_eq!(Staking::balance(&alice()), 41);
+			assert_eq!(Staking::balance(&bob()), 69);
 		});
 
 		Executor::new().call(&mut t, COMPACT_CODE, "execute_block", &block2().0).unwrap();
 
 		runtime_io::with_externalities(&mut t, || {
-			assert_eq!(Staking::balance(&Alice), 30);
-			assert_eq!(Staking::balance(&Bob), 78);
+			assert_eq!(Staking::balance(&alice()), 30);
+			assert_eq!(Staking::balance(&bob()), 78);
 		});
 	}
 
@@ -257,22 +265,22 @@ mod tests {
 		WasmExecutor.call(&mut t, COMPACT_CODE, "execute_block", &block1().0).unwrap();
 
 		runtime_io::with_externalities(&mut t, || {
-			assert_eq!(Staking::balance(&Alice), 41);
-			assert_eq!(Staking::balance(&Bob), 69);
+			assert_eq!(Staking::balance(&alice()), 41);
+			assert_eq!(Staking::balance(&bob()), 69);
 		});
 
 		WasmExecutor.call(&mut t, COMPACT_CODE, "execute_block", &block2().0).unwrap();
 
 		runtime_io::with_externalities(&mut t, || {
-			assert_eq!(Staking::balance(&Alice), 30);
-			assert_eq!(Staking::balance(&Bob), 78);
+			assert_eq!(Staking::balance(&alice()), 30);
+			assert_eq!(Staking::balance(&bob()), 78);
 		});
 	}
 
 	#[test]
 	fn panic_execution_gives_error() {
 		let mut t: TestExternalities = map![
-			twox_128(&<staking::FreeBalance<Concrete>>::key_for(*Alice)).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0],
+			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![70u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
@@ -288,7 +296,7 @@ mod tests {
 	#[test]
 	fn successful_execution_gives_ok() {
 		let mut t: TestExternalities = map![
-			twox_128(&<staking::FreeBalance<Concrete>>::key_for(*Alice)).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0],
+			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(&<system::BlockHash<Concrete>>::key_for(0)).to_vec() => vec![0u8; 32]
@@ -301,8 +309,8 @@ mod tests {
 		assert!(r.is_ok());
 
 		runtime_io::with_externalities(&mut t, || {
-			assert_eq!(Staking::balance(&Alice), 42);
-			assert_eq!(Staking::balance(&Bob), 69);
+			assert_eq!(Staking::balance(&alice()), 42);
+			assert_eq!(Staking::balance(&bob()), 69);
 		});
 	}
 }

demo/executor/src/lib.rs

@@ -33,7 +33,7 @@ pub type BlockNumber = u64;
 
 /// Alias to Ed25519 pubkey that identifies an account on the relay chain. This will almost
 /// certainly continue to be the same as the substrate's `AuthorityId`.
-pub type AccountId = primitives::AuthorityId;
+pub type AccountId = ::primitives::H256;
 
 /// Balance of an account.
 pub type Balance = u64;

demo/primitives/src/lib.rs

@@ -22,11 +22,18 @@
 extern crate substrate_runtime_io as runtime_io;
 
 #[macro_use]
-extern crate substrate_runtime_support as runtime_support;
+extern crate substrate_runtime_support;
 
 #[macro_use]
 extern crate substrate_runtime_primitives as runtime_primitives;
 
+#[cfg(feature = "std")]
+#[macro_use]
+extern crate serde_derive;
+
+#[cfg(feature = "std")]
+extern crate serde;
+
 extern crate substrate_runtime_std as rstd;
 extern crate substrate_runtime_consensus as consensus;
 extern crate substrate_runtime_council as council;
@@ -39,10 +46,9 @@ extern crate substrate_runtime_timestamp as timestamp;
 extern crate demo_primitives;
 
 use rstd::prelude::*;
-use runtime_io::BlakeTwo256;
 use demo_primitives::{AccountId, Balance, BlockNumber, Hash, Index, SessionKey, Signature};
 use runtime_primitives::generic;
-use runtime_primitives::traits::{Identity, HasPublicAux};
+use runtime_primitives::traits::{Convert, HasPublicAux, BlakeTwo256};
 
 #[cfg(any(feature = "std", test))]
 pub use runtime_primitives::BuildExternalities;
@@ -61,7 +67,7 @@ impl system::Trait for Concrete {
 	type Hashing = BlakeTwo256;
 	type Digest = generic::Digest<Vec<u8>>;
 	type AccountId = AccountId;
-	type Header = generic::Header<BlockNumber, Hash, Vec<u8>>;
+	type Header = generic::Header<BlockNumber, BlakeTwo256, Vec<u8>>;
 }
 
 /// System module for this concrete runtime.
@@ -84,8 +90,16 @@ impl timestamp::Trait for Concrete {
 /// Timestamp module for this concrete runtime.
 pub type Timestamp = timestamp::Module<Concrete>;
 
+/// Session key conversion.
+pub struct SessionKeyConversion;
+impl Convert<AccountId, SessionKey> for SessionKeyConversion {
+	fn convert(a: AccountId) -> SessionKey {
+		a.0
+	}
+}
+
 impl session::Trait for Concrete {
-	type ConvertAccountIdToSessionKey = Identity;
+	type ConvertAccountIdToSessionKey = SessionKeyConversion;
 }
 
 /// Session module for this concrete runtime.
@@ -114,6 +128,8 @@ pub type Council = council::Module<Concrete>;
 pub type CouncilVoting = council::voting::Module<Concrete>;
 
 impl_outer_dispatch! {
+	#[derive(Clone, PartialEq, Eq)]
+	#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))]
 	pub enum Call where aux: <Concrete as HasPublicAux>::PublicAux {
 		Consensus = 0,
 		Session = 1,
@@ -124,6 +140,8 @@ impl_outer_dispatch! {
 		CouncilVoting = 7,
 	}
 
+	#[derive(Clone, PartialEq, Eq)]
+	#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))]
 	pub enum PrivCall {
 		Consensus = 0,
 		Session = 1,
@@ -135,9 +153,9 @@ impl_outer_dispatch! {
 }
 
 /// Block header type as expected by this runtime.
-pub type Header = generic::Header<BlockNumber, Hash, Vec<u8>>;
+pub type Header = generic::Header<BlockNumber, BlakeTwo256, Vec<u8>>;
 /// Block type as expected by this runtime.
-pub type Block = generic::Block<BlockNumber, Hash, Vec<u8>, AccountId, Index, Call, Signature>;
+pub type Block = generic::Block<Header, UncheckedExtrinsic>;
 /// Unchecked extrinsic type as expected by this runtime.
 pub type UncheckedExtrinsic = generic::UncheckedExtrinsic<AccountId, Index, Call, Signature>;
 /// Extrinsic type as expected by this runtime.