From 1e7830d53a8c04e8d3cd8a4ffe19a34390a91f06 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 1 Jul 2021 16:25:37 +1000 Subject: [PATCH 1/6] Use hardware acceleration for SHA256 --- Cargo.lock | 52 +++++++++++-------- .../src/compute_shuffled_index.rs | 10 ++-- .../swap_or_not_shuffle/src/shuffle_list.rs | 13 ++--- consensus/tree_hash/src/lib.rs | 9 +--- consensus/tree_hash/src/merkle_hasher.rs | 16 +++--- consensus/tree_hash/src/merkleize_padded.rs | 6 +-- crypto/eth2_hashing/Cargo.toml | 7 +-- crypto/eth2_hashing/src/lib.rs | 40 +++----------- 8 files changed, 60 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87bd7e0d601..ddc231b2512 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1222,6 +1222,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +[[package]] +name = "cpufeatures" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" +dependencies = [ + "libc", +] + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -1495,7 +1504,7 @@ dependencies = [ "hex", "reqwest", "serde_json", - "sha2 0.9.3", + "sha2 0.9.5", "tree_hash", "types", ] @@ -1662,7 +1671,7 @@ dependencies = [ "parking_lot", "rand 0.7.3", "rlp 0.5.0", - "sha2 0.9.3", + "sha2 0.9.5", "smallvec", "tokio 1.5.0", "tokio-stream", @@ -1694,7 +1703,7 @@ dependencies = [ "parking_lot", "rand 0.7.3", "rlp 0.5.0", - "sha2 0.9.3", + "sha2 0.9.5", "smallvec", "tokio 1.5.0", "tokio-stream", @@ -1741,7 +1750,7 @@ dependencies = [ "ed25519", "rand 0.7.3", "serde", - "sha2 0.9.3", + "sha2 0.9.5", "zeroize", ] @@ -1972,9 +1981,8 @@ name = "eth2_hashing" version = "0.1.1" dependencies = [ "lazy_static", - "ring", "rustc-hex", - "sha2 0.9.3", + "sha2 0.9.5", "wasm-bindgen-test", ] @@ -2001,7 +2009,7 @@ dependencies = [ "hex", "num-bigint-dig", "ring", - "sha2 0.9.3", + "sha2 0.9.5", "zeroize", ] @@ -2021,7 +2029,7 @@ dependencies = [ "serde", "serde_json", "serde_repr", - "sha2 0.9.3", + "sha2 0.9.5", "tempfile", "unicode-normalization", "uuid", @@ -2056,7 +2064,7 @@ dependencies = [ "regex", "serde", "serde_derive", - "sha2 0.9.3", + "sha2 0.9.5", "slog", "slog-async", "slog-term", @@ -3283,7 +3291,7 @@ dependencies = [ "cfg-if 1.0.0", "ecdsa", "elliptic-curve", - "sha2 0.9.3", + "sha2 0.9.5", ] [[package]] @@ -3419,9 +3427,9 @@ checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" [[package]] name = "libc" -version = "0.2.94" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" +checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" [[package]] name = "libflate" @@ -3513,7 +3521,7 @@ dependencies = [ "rand 0.7.3", "ring", "rw-stream-sink", - "sha2 0.9.3", + "sha2 0.9.5", "smallvec", "thiserror", "unsigned-varint 0.7.0", @@ -3552,7 +3560,7 @@ dependencies = [ "prost-build", "rand 0.7.3", "regex", - "sha2 0.9.3", + "sha2 0.9.5", "smallvec", "unsigned-varint 0.7.0", "wasm-timer", @@ -3607,7 +3615,7 @@ dependencies = [ "prost", "prost-build", "rand 0.7.3", - "sha2 0.9.3", + "sha2 0.9.5", "snow", "static_assertions", "x25519-dalek", @@ -4082,7 +4090,7 @@ dependencies = [ "digest 0.9.0", "generic-array 0.14.4", "multihash-derive", - "sha2 0.9.3", + "sha2 0.9.5", "unsigned-varint 0.5.1", ] @@ -5520,7 +5528,7 @@ dependencies = [ "hmac 0.10.1", "pbkdf2 0.6.0", "salsa20", - "sha2 0.9.3", + "sha2 0.9.5", ] [[package]] @@ -5744,13 +5752,13 @@ dependencies = [ [[package]] name = "sha2" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" +checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", - "cpuid-bool 0.1.2", + "cpufeatures", "digest 0.9.0", "opaque-debug 0.3.0", ] @@ -6055,7 +6063,7 @@ dependencies = [ "rand_core 0.5.1", "ring", "rustc_version 0.2.3", - "sha2 0.9.3", + "sha2 0.9.5", "subtle 2.4.0", "x25519-dalek", ] @@ -6533,7 +6541,7 @@ dependencies = [ "pbkdf2 0.4.0", "rand 0.7.3", "rustc-hash", - "sha2 0.9.3", + "sha2 0.9.5", "thiserror", "unicode-normalization", "zeroize", diff --git a/consensus/swap_or_not_shuffle/src/compute_shuffled_index.rs b/consensus/swap_or_not_shuffle/src/compute_shuffled_index.rs index 6f863525b22..8330fc313c6 100644 --- a/consensus/swap_or_not_shuffle/src/compute_shuffled_index.rs +++ b/consensus/swap_or_not_shuffle/src/compute_shuffled_index.rs @@ -1,5 +1,5 @@ use crate::Hash256; -use eth2_hashing::{Context, SHA256}; +use eth2_hashing::{Digest, Sha256}; use std::cmp::max; /// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. @@ -54,7 +54,7 @@ fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize } fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Hash256 { - let mut context = Context::new(&SHA256); + let mut context = Sha256::new(); context.update(seed); context.update(&[round]); @@ -64,17 +64,17 @@ fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Hash */ context.update(&(position / 256).to_le_bytes()[0..4]); - let digest = context.finish(); + let digest = context.finalize(); Hash256::from_slice(digest.as_ref()) } fn hash_with_round(seed: &[u8], round: u8) -> Hash256 { - let mut context = Context::new(&SHA256); + let mut context = Sha256::new(); context.update(seed); context.update(&[round]); - let digest = context.finish(); + let digest = context.finalize(); Hash256::from_slice(digest.as_ref()) } diff --git a/consensus/swap_or_not_shuffle/src/shuffle_list.rs b/consensus/swap_or_not_shuffle/src/shuffle_list.rs index 41e91beb23b..edc6dd6377c 100644 --- a/consensus/swap_or_not_shuffle/src/shuffle_list.rs +++ b/consensus/swap_or_not_shuffle/src/shuffle_list.rs @@ -1,5 +1,5 @@ use crate::Hash256; -use eth2_hashing::{Context, SHA256}; +use eth2_hashing::hash_fixed; use std::mem; const SEED_SIZE: usize = 32; @@ -31,12 +31,10 @@ impl Buf { /// Returns the new pivot. It is "raw" because it has not modulo the list size (this must be /// done by the caller). fn raw_pivot(&self) -> u64 { - let mut context = Context::new(&SHA256); - context.update(&self.0[0..PIVOT_VIEW_SIZE]); - let digest = context.finish(); + let digest = hash_fixed(&self.0[0..PIVOT_VIEW_SIZE]); let mut bytes = [0; mem::size_of::()]; - bytes[..].copy_from_slice(&digest.as_ref()[0..mem::size_of::()]); + bytes[..].copy_from_slice(&digest[0..mem::size_of::()]); u64::from_le_bytes(bytes) } @@ -47,10 +45,7 @@ impl Buf { /// Hash the entire buffer. fn hash(&self) -> Hash256 { - let mut context = Context::new(&SHA256); - context.update(&self.0[..]); - let digest = context.finish(); - Hash256::from_slice(digest.as_ref()) + Hash256::from_slice(&hash_fixed(&self.0)) } } diff --git a/consensus/tree_hash/src/lib.rs b/consensus/tree_hash/src/lib.rs index 7008e0068dd..b18ba4db2be 100644 --- a/consensus/tree_hash/src/lib.rs +++ b/consensus/tree_hash/src/lib.rs @@ -7,8 +7,7 @@ pub use merkle_hasher::{Error, MerkleHasher}; pub use merkleize_padded::merkleize_padded; pub use merkleize_standard::merkleize_standard; -use eth2_hashing::{Context, SHA256}; -use eth2_hashing::{ZERO_HASHES, ZERO_HASHES_MAX_INDEX}; +use eth2_hashing::{hash_fixed, ZERO_HASHES, ZERO_HASHES_MAX_INDEX}; pub const BYTES_PER_CHUNK: usize = 32; pub const HASHSIZE: usize = 32; @@ -39,11 +38,7 @@ pub fn merkle_root(bytes: &[u8], minimum_leaf_count: usize) -> Hash256 { let mut leaves = [0; HASHSIZE * 2]; leaves[0..bytes.len()].copy_from_slice(bytes); - let mut context = Context::new(&SHA256); - context.update(&leaves); - let digest = context.finish(); - - Hash256::from_slice(digest.as_ref()) + Hash256::from_slice(&hash_fixed(&leaves)) } else { // If there are 3 or more leaves, use `MerkleHasher`. let mut hasher = MerkleHasher::with_leaves(leaves); diff --git a/consensus/tree_hash/src/merkle_hasher.rs b/consensus/tree_hash/src/merkle_hasher.rs index 7a313710510..4180adffbab 100644 --- a/consensus/tree_hash/src/merkle_hasher.rs +++ b/consensus/tree_hash/src/merkle_hasher.rs @@ -1,5 +1,5 @@ use crate::{get_zero_hash, Hash256, HASHSIZE}; -use eth2_hashing::{Context, Digest, SHA256}; +use eth2_hashing::{Digest, Sha256}; use smallvec::{smallvec, SmallVec}; use std::mem; @@ -15,7 +15,7 @@ pub enum Error { /// /// Should be used as a left or right value for some node. enum Preimage<'a> { - Digest(Digest), + Digest([u8; 32]), Slice(&'a [u8]), } @@ -32,7 +32,7 @@ impl<'a> Preimage<'a> { /// A node that has had a left child supplied, but not a right child. struct HalfNode { /// The hasher context. - context: Context, + context: Sha256, /// The tree id of the node. The root node has in id of `1` and ids increase moving down the /// tree from left to right. id: usize, @@ -41,7 +41,7 @@ struct HalfNode { impl HalfNode { /// Create a new half-node from the given `left` value. fn new(id: usize, left: Preimage) -> Self { - let mut context = Context::new(&SHA256); + let mut context = Sha256::new(); context.update(left.as_bytes()); Self { context, id } @@ -49,9 +49,9 @@ impl HalfNode { /// Complete the half-node by providing a `right` value. Returns a digest of the left and right /// nodes. - fn finish(mut self, right: Preimage) -> Digest { + fn finish(mut self, right: Preimage) -> [u8; 32] { self.context.update(right.as_bytes()); - self.context.finish() + self.context.finalize().into() } } @@ -124,7 +124,7 @@ pub struct MerkleHasher { /// Stores the nodes that are half-complete and awaiting a right node. /// /// A smallvec of size 8 means we can hash a tree with 256 leaves without allocating on the - /// heap. Each half-node is 224 bytes, so this smallvec may store 1,792 bytes on the stack. + /// heap. Each half-node is 120 bytes, so this smallvec may store 960 bytes on the stack. half_nodes: SmallVec8, /// The depth of the tree that will be produced. /// @@ -368,7 +368,7 @@ mod test { fn context_size() { assert_eq!( mem::size_of::(), - 216 + 8, + 112 + 8, "Halfnode size should be as expected" ); } diff --git a/consensus/tree_hash/src/merkleize_padded.rs b/consensus/tree_hash/src/merkleize_padded.rs index 18beb536290..02c8af1b36f 100644 --- a/consensus/tree_hash/src/merkleize_padded.rs +++ b/consensus/tree_hash/src/merkleize_padded.rs @@ -1,5 +1,5 @@ use super::{get_zero_hash, Hash256, BYTES_PER_CHUNK}; -use eth2_hashing::{hash, hash32_concat}; +use eth2_hashing::{hash32_concat, hash_fixed}; /// Merkleize `bytes` and return the root, optionally padding the tree out to `min_leaves` number of /// leaves. @@ -79,7 +79,7 @@ pub fn merkleize_padded(bytes: &[u8], min_leaves: usize) -> Hash256 { // Hash two chunks, creating a parent chunk. let hash = match bytes.get(start..start + BYTES_PER_CHUNK * 2) { // All bytes are available, hash as usual. - Some(slice) => hash(slice), + Some(slice) => hash_fixed(slice), // Unable to get all the bytes, get a small slice and pad it out. None => { let mut preimage = bytes @@ -87,7 +87,7 @@ pub fn merkleize_padded(bytes: &[u8], min_leaves: usize) -> Hash256 { .expect("`i` can only be larger than zero if there are bytes to read") .to_vec(); preimage.resize(BYTES_PER_CHUNK * 2, 0); - hash(&preimage) + hash_fixed(&preimage) } }; diff --git a/crypto/eth2_hashing/Cargo.toml b/crypto/eth2_hashing/Cargo.toml index 7fd4bbdd7be..8fe340983db 100644 --- a/crypto/eth2_hashing/Cargo.toml +++ b/crypto/eth2_hashing/Cargo.toml @@ -8,12 +8,7 @@ description = "Hashing primitives used in Ethereum 2.0" [dependencies] lazy_static = { version = "1.4.0", optional = true } - -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -ring = "0.16.19" - -[target.'cfg(target_arch = "wasm32")'.dependencies] -sha2 = "0.9.1" +sha2 = "0.9.5" [dev-dependencies] rustc-hex = "2.1.0" diff --git a/crypto/eth2_hashing/src/lib.rs b/crypto/eth2_hashing/src/lib.rs index e2f29c03643..078ee7dafac 100644 --- a/crypto/eth2_hashing/src/lib.rs +++ b/crypto/eth2_hashing/src/lib.rs @@ -1,45 +1,22 @@ -//! Provides a simple hash function utilizing `ring::digest::SHA256`. +//! Provides a simple hash function utilizing `sha2::Sha256`. //! //! The purpose of this crate is to provide an abstraction to whatever hash function Ethereum //! 2.0 is using. The hash function has been subject to change during the specification process, so //! defining it once in this crate makes it easy to replace. -#[cfg(not(target_arch = "wasm32"))] -pub use ring::digest::{digest, Context, Digest, SHA256}; - -#[cfg(target_arch = "wasm32")] -use sha2::{Digest, Sha256}; +pub use sha2::{Digest, Sha256}; #[cfg(feature = "zero_hash_cache")] use lazy_static::lazy_static; /// Returns the digest of `input`. -/// -/// Uses `ring::digest::SHA256`. pub fn hash(input: &[u8]) -> Vec { - #[cfg(not(target_arch = "wasm32"))] - let h = digest(&SHA256, input).as_ref().into(); - - #[cfg(target_arch = "wasm32")] - let h = Sha256::digest(input).as_ref().into(); - - h + Sha256::digest(input).into_iter().collect() } -/// Compute the hash of two slices concatenated. -/// -/// # Panics -/// -/// Will panic if either `h1` or `h2` are not 32 bytes in length. -#[cfg(not(target_arch = "wasm32"))] -pub fn hash32_concat(h1: &[u8], h2: &[u8]) -> [u8; 32] { - let mut context = Context::new(&SHA256); - context.update(h1); - context.update(h2); - - let mut output = [0; 32]; - output[..].copy_from_slice(context.finish().as_ref()); - output +/// Hash function returning a fixed-size array (to save on allocations). +pub fn hash_fixed(input: &[u8]) -> [u8; 32] { + Sha256::digest(input).into() } /// Compute the hash of two slices concatenated. @@ -47,15 +24,12 @@ pub fn hash32_concat(h1: &[u8], h2: &[u8]) -> [u8; 32] { /// # Panics /// /// Will panic if either `h1` or `h2` are not 32 bytes in length. -#[cfg(target_arch = "wasm32")] pub fn hash32_concat(h1: &[u8], h2: &[u8]) -> [u8; 32] { let mut preimage = [0; 64]; preimage[0..32].copy_from_slice(h1); preimage[32..64].copy_from_slice(h2); - let mut output = [0; 32]; - output[..].copy_from_slice(&hash(&preimage)); - output + hash_fixed(&preimage) } /// The max index that can be used with `ZERO_HASHES`. From ac114e160e3ed25532da45628819009e3c6b21e3 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 2 Jul 2021 15:56:38 +1000 Subject: [PATCH 2/6] Implement runtime dispatch --- Cargo.lock | 2 + .../src/compute_shuffled_index.rs | 6 +- consensus/tree_hash/src/merkle_hasher.rs | 10 +- crypto/eth2_hashing/Cargo.toml | 2 + crypto/eth2_hashing/src/lib.rs | 159 +++++++++++++++++- 5 files changed, 167 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ddc231b2512..ee66f4977ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1980,7 +1980,9 @@ dependencies = [ name = "eth2_hashing" version = "0.1.1" dependencies = [ + "cpufeatures", "lazy_static", + "ring", "rustc-hex", "sha2 0.9.5", "wasm-bindgen-test", diff --git a/consensus/swap_or_not_shuffle/src/compute_shuffled_index.rs b/consensus/swap_or_not_shuffle/src/compute_shuffled_index.rs index 8330fc313c6..f43edfe8644 100644 --- a/consensus/swap_or_not_shuffle/src/compute_shuffled_index.rs +++ b/consensus/swap_or_not_shuffle/src/compute_shuffled_index.rs @@ -1,5 +1,5 @@ use crate::Hash256; -use eth2_hashing::{Digest, Sha256}; +use eth2_hashing::{Context, Sha256Context}; use std::cmp::max; /// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy. @@ -54,7 +54,7 @@ fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize } fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Hash256 { - let mut context = Sha256::new(); + let mut context = Context::new(); context.update(seed); context.update(&[round]); @@ -69,7 +69,7 @@ fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Hash } fn hash_with_round(seed: &[u8], round: u8) -> Hash256 { - let mut context = Sha256::new(); + let mut context = Context::new(); context.update(seed); context.update(&[round]); diff --git a/consensus/tree_hash/src/merkle_hasher.rs b/consensus/tree_hash/src/merkle_hasher.rs index 4180adffbab..ce10ec6ab42 100644 --- a/consensus/tree_hash/src/merkle_hasher.rs +++ b/consensus/tree_hash/src/merkle_hasher.rs @@ -1,5 +1,5 @@ use crate::{get_zero_hash, Hash256, HASHSIZE}; -use eth2_hashing::{Digest, Sha256}; +use eth2_hashing::{Context, Sha256Context}; use smallvec::{smallvec, SmallVec}; use std::mem; @@ -32,7 +32,7 @@ impl<'a> Preimage<'a> { /// A node that has had a left child supplied, but not a right child. struct HalfNode { /// The hasher context. - context: Sha256, + context: Context, /// The tree id of the node. The root node has in id of `1` and ids increase moving down the /// tree from left to right. id: usize, @@ -41,7 +41,7 @@ struct HalfNode { impl HalfNode { /// Create a new half-node from the given `left` value. fn new(id: usize, left: Preimage) -> Self { - let mut context = Sha256::new(); + let mut context = Context::new(); context.update(left.as_bytes()); Self { context, id } @@ -124,7 +124,7 @@ pub struct MerkleHasher { /// Stores the nodes that are half-complete and awaiting a right node. /// /// A smallvec of size 8 means we can hash a tree with 256 leaves without allocating on the - /// heap. Each half-node is 120 bytes, so this smallvec may store 960 bytes on the stack. + /// heap. Each half-node is 232 bytes, so this smallvec may store 1856 bytes on the stack. half_nodes: SmallVec8, /// The depth of the tree that will be produced. /// @@ -368,7 +368,7 @@ mod test { fn context_size() { assert_eq!( mem::size_of::(), - 112 + 8, + 232, "Halfnode size should be as expected" ); } diff --git a/crypto/eth2_hashing/Cargo.toml b/crypto/eth2_hashing/Cargo.toml index 8fe340983db..2e29f69d8aa 100644 --- a/crypto/eth2_hashing/Cargo.toml +++ b/crypto/eth2_hashing/Cargo.toml @@ -8,7 +8,9 @@ description = "Hashing primitives used in Ethereum 2.0" [dependencies] lazy_static = { version = "1.4.0", optional = true } +ring = "0.16.19" sha2 = "0.9.5" +cpufeatures = "0.1.5" [dev-dependencies] rustc-hex = "2.1.0" diff --git a/crypto/eth2_hashing/src/lib.rs b/crypto/eth2_hashing/src/lib.rs index 078ee7dafac..2641ee131bb 100644 --- a/crypto/eth2_hashing/src/lib.rs +++ b/crypto/eth2_hashing/src/lib.rs @@ -4,19 +4,23 @@ //! 2.0 is using. The hash function has been subject to change during the specification process, so //! defining it once in this crate makes it easy to replace. -pub use sha2::{Digest, Sha256}; +pub use self::DynamicContext as Context; +use sha2::Digest; #[cfg(feature = "zero_hash_cache")] use lazy_static::lazy_static; +/// Length of a SHA256 hash in bytes. +pub const HASH_LEN: usize = 32; + /// Returns the digest of `input`. pub fn hash(input: &[u8]) -> Vec { - Sha256::digest(input).into_iter().collect() + DynamicImpl::best().hash(input) } /// Hash function returning a fixed-size array (to save on allocations). -pub fn hash_fixed(input: &[u8]) -> [u8; 32] { - Sha256::digest(input).into() +pub fn hash_fixed(input: &[u8]) -> [u8; HASH_LEN] { + DynamicImpl::best().hash_fixed(input) } /// Compute the hash of two slices concatenated. @@ -32,6 +36,153 @@ pub fn hash32_concat(h1: &[u8], h2: &[u8]) -> [u8; 32] { hash_fixed(&preimage) } +pub trait Sha256Context { + fn new() -> Self; + + fn update(&mut self, bytes: &[u8]); + + fn finalize(self) -> [u8; HASH_LEN]; +} + +pub trait Sha256 { + type Context: Sha256Context; + + fn hash(&self, input: &[u8]) -> Vec; + + fn hash_fixed(&self, input: &[u8]) -> [u8; HASH_LEN]; +} + +/// Implementation of SHA256 using the `sha2` crate (fastest on CPUs with SHA extensions). +struct Sha2CrateImpl; + +impl Sha256Context for sha2::Sha256 { + fn new() -> Self { + sha2::Digest::new() + } + + fn update(&mut self, bytes: &[u8]) { + sha2::Digest::update(self, bytes) + } + + fn finalize(self) -> [u8; HASH_LEN] { + sha2::Digest::finalize(self).into() + } +} + +impl Sha256 for Sha2CrateImpl { + type Context = sha2::Sha256; + + fn hash(&self, input: &[u8]) -> Vec { + Self::Context::digest(input).into_iter().collect() + } + + fn hash_fixed(&self, input: &[u8]) -> [u8; HASH_LEN] { + Self::Context::digest(input).into() + } +} + +/// Implementation of SHA256 using the `ring` crate (fastest on CPUs without SHA extensions). +pub struct RingImpl; + +impl Sha256Context for ring::digest::Context { + fn new() -> Self { + Self::new(&ring::digest::SHA256) + } + + fn update(&mut self, bytes: &[u8]) { + self.update(bytes) + } + + fn finalize(self) -> [u8; HASH_LEN] { + let mut output = [0; HASH_LEN]; + output.copy_from_slice(self.finish().as_ref()); + output + } +} + +impl Sha256 for RingImpl { + type Context = ring::digest::Context; + + fn hash(&self, input: &[u8]) -> Vec { + ring::digest::digest(&ring::digest::SHA256, input) + .as_ref() + .into() + } + + fn hash_fixed(&self, input: &[u8]) -> [u8; HASH_LEN] { + let mut ctxt = Self::Context::new(&ring::digest::SHA256); + ctxt.update(input); + ctxt.finalize() + } +} + +// Inspired by the runtime-detection within the `sha2` crate itself. +cpufeatures::new!(x86_sha_extensions, "sha", "sse2", "ssse3", "sse4.1"); + +pub enum DynamicImpl { + Sha2, + Ring, +} + +impl DynamicImpl { + #[inline(always)] + pub fn best() -> Self { + if x86_sha_extensions::get() { + Self::Sha2 + } else { + Self::Ring + } + } +} + +impl Sha256 for DynamicImpl { + type Context = DynamicContext; + + #[inline(always)] + fn hash(&self, input: &[u8]) -> Vec { + match self { + Self::Sha2 => Sha2CrateImpl.hash(input), + Self::Ring => RingImpl.hash(input), + } + } + + #[inline(always)] + fn hash_fixed(&self, input: &[u8]) -> [u8; HASH_LEN] { + match self { + Self::Sha2 => Sha2CrateImpl.hash_fixed(input), + Self::Ring => RingImpl.hash_fixed(input), + } + } +} + +pub enum DynamicContext { + Sha2(sha2::Sha256), + Ring(ring::digest::Context), +} + +impl Sha256Context for DynamicContext { + fn new() -> Self { + match DynamicImpl::best() { + DynamicImpl::Sha2 => Self::Sha2(Sha256Context::new()), + DynamicImpl::Ring => Self::Ring(Sha256Context::new()), + } + } + + fn update(&mut self, bytes: &[u8]) { + match self { + Self::Sha2(ctxt) => Sha256Context::update(ctxt, bytes), + Self::Ring(ctxt) => Sha256Context::update(ctxt, bytes), + } + } + + fn finalize(self) -> [u8; HASH_LEN] { + match self { + Self::Sha2(ctxt) => Sha256Context::finalize(ctxt), + Self::Ring(ctxt) => Sha256Context::finalize(ctxt), + } + } +} + /// The max index that can be used with `ZERO_HASHES`. #[cfg(feature = "zero_hash_cache")] pub const ZERO_HASHES_MAX_INDEX: usize = 48; From cfa67bd55adc049c5a6639f5ebe10e0dae975257 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 2 Jul 2021 16:17:07 +1000 Subject: [PATCH 3/6] Appease Clippy --- consensus/tree_hash/src/merkle_hasher.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/consensus/tree_hash/src/merkle_hasher.rs b/consensus/tree_hash/src/merkle_hasher.rs index ce10ec6ab42..c7fdb174617 100644 --- a/consensus/tree_hash/src/merkle_hasher.rs +++ b/consensus/tree_hash/src/merkle_hasher.rs @@ -1,5 +1,5 @@ use crate::{get_zero_hash, Hash256, HASHSIZE}; -use eth2_hashing::{Context, Sha256Context}; +use eth2_hashing::{Context, Sha256Context, HASH_LEN}; use smallvec::{smallvec, SmallVec}; use std::mem; @@ -15,7 +15,7 @@ pub enum Error { /// /// Should be used as a left or right value for some node. enum Preimage<'a> { - Digest([u8; 32]), + Digest([u8; HASH_LEN]), Slice(&'a [u8]), } @@ -49,9 +49,9 @@ impl HalfNode { /// Complete the half-node by providing a `right` value. Returns a digest of the left and right /// nodes. - fn finish(mut self, right: Preimage) -> [u8; 32] { + fn finish(mut self, right: Preimage) -> [u8; HASH_LEN] { self.context.update(right.as_bytes()); - self.context.finalize().into() + self.context.finalize() } } From 1f17bc5b003661a0ed0b285d18f053ff284352f7 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 2 Jul 2021 16:47:07 +1000 Subject: [PATCH 4/6] Add comments, `cfg` out the x86 stuff --- crypto/eth2_hashing/src/lib.rs | 42 ++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/crypto/eth2_hashing/src/lib.rs b/crypto/eth2_hashing/src/lib.rs index 2641ee131bb..9f61e273554 100644 --- a/crypto/eth2_hashing/src/lib.rs +++ b/crypto/eth2_hashing/src/lib.rs @@ -1,8 +1,11 @@ -//! Provides a simple hash function utilizing `sha2::Sha256`. +//! Optimized SHA256 for use in Ethereum 2.0. //! -//! The purpose of this crate is to provide an abstraction to whatever hash function Ethereum -//! 2.0 is using. The hash function has been subject to change during the specification process, so -//! defining it once in this crate makes it easy to replace. +//! The initial purpose of this crate was to provide an abstraction over the hash function used in +//! Ethereum 2.0. The hash function changed during the specification process, so defining it once in +//! this crate made it easy to replace. +//! +//! Now this crate serves primarily as a wrapper over two SHA256 crates: `sha2` and `ring` – +//! which it switches between at runtime based on the availability of SHA intrinsics. pub use self::DynamicContext as Context; use sha2::Digest; @@ -13,12 +16,14 @@ use lazy_static::lazy_static; /// Length of a SHA256 hash in bytes. pub const HASH_LEN: usize = 32; -/// Returns the digest of `input`. +/// Returns the digest of `input` using the best available implementation. pub fn hash(input: &[u8]) -> Vec { DynamicImpl::best().hash(input) } /// Hash function returning a fixed-size array (to save on allocations). +/// +/// Uses the best available implementation based on CPU features. pub fn hash_fixed(input: &[u8]) -> [u8; HASH_LEN] { DynamicImpl::best().hash_fixed(input) } @@ -36,6 +41,7 @@ pub fn hash32_concat(h1: &[u8], h2: &[u8]) -> [u8; 32] { hash_fixed(&preimage) } +/// Context trait for abstracting over implementation contexts. pub trait Sha256Context { fn new() -> Self; @@ -44,6 +50,7 @@ pub trait Sha256Context { fn finalize(self) -> [u8; HASH_LEN]; } +/// Top-level trait implemented by both `sha2` and `ring` implementations. pub trait Sha256 { type Context: Sha256Context; @@ -116,18 +123,32 @@ impl Sha256 for RingImpl { } } -// Inspired by the runtime-detection within the `sha2` crate itself. -cpufeatures::new!(x86_sha_extensions, "sha", "sse2", "ssse3", "sse4.1"); - +/// Default dynamic implementation that switches between available implementations. pub enum DynamicImpl { Sha2, Ring, } +// Runtime latch for detecting the availability of SHA extensions on x86_64. +// +// Inspired by the runtime switch within the `sha2` crate itself. +#[cfg(target_arch = "x86_64")] +cpufeatures::new!(x86_sha_extensions, "sha", "sse2", "ssse3", "sse4.1"); + +#[inline(always)] +pub fn have_sha_extensions() -> bool { + #[cfg(target_arch = "x86_64")] + return x86_sha_extensions::get(); + + #[cfg(not(target_arch = "x86_64"))] + return false; +} + impl DynamicImpl { + /// Choose the best available implementation based on the currently executing CPU. #[inline(always)] pub fn best() -> Self { - if x86_sha_extensions::get() { + if have_sha_extensions() { Self::Sha2 } else { Self::Ring @@ -155,6 +176,9 @@ impl Sha256 for DynamicImpl { } } +/// Context encapsulating all implemenation contexts. +/// +/// This enum ends up being 8 bytes larger than the largest inner context. pub enum DynamicContext { Sha2(sha2::Sha256), Ring(ring::digest::Context), From d9242f32bb5d25f76d48d17e3c9b19e93967274b Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 12 Jul 2021 18:30:32 +1000 Subject: [PATCH 5/6] Remove intermediate value in hash32_concat --- crypto/eth2_hashing/src/lib.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/crypto/eth2_hashing/src/lib.rs b/crypto/eth2_hashing/src/lib.rs index 9f61e273554..c5c034640b6 100644 --- a/crypto/eth2_hashing/src/lib.rs +++ b/crypto/eth2_hashing/src/lib.rs @@ -29,16 +29,11 @@ pub fn hash_fixed(input: &[u8]) -> [u8; HASH_LEN] { } /// Compute the hash of two slices concatenated. -/// -/// # Panics -/// -/// Will panic if either `h1` or `h2` are not 32 bytes in length. pub fn hash32_concat(h1: &[u8], h2: &[u8]) -> [u8; 32] { - let mut preimage = [0; 64]; - preimage[0..32].copy_from_slice(h1); - preimage[32..64].copy_from_slice(h2); - - hash_fixed(&preimage) + let mut ctxt = DynamicContext::new(); + ctxt.update(h1); + ctxt.update(h2); + ctxt.finalize() } /// Context trait for abstracting over implementation contexts. From dcf52b4ec625e6ab05a87e637edc80fffcd4e3d9 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 12 Jul 2021 18:31:18 +1000 Subject: [PATCH 6/6] Print SHA256 acceleration info in --version --- lighthouse/Cargo.toml | 3 +-- lighthouse/src/main.rs | 18 +++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index ff6c6954711..925ce855c44 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -16,8 +16,6 @@ modern = ["bls/supranational-force-adx"] milagro = ["bls/milagro"] # Support minimal spec (used for testing only). spec-minimal = [] -# Support spec v0.12 (used by Medalla testnet). -spec-v12 = [] [dependencies] beacon_node = { "path" = "../beacon_node" } @@ -26,6 +24,7 @@ slog = { version = "2.5.2", features = ["max_level_trace"] } sloggers = "1.0.1" types = { "path" = "../consensus/types" } bls = { path = "../crypto/bls" } +eth2_hashing = "0.1.0" clap = "2.33.3" env_logger = "0.8.2" logging = { path = "../common/logging" } diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index e49e29f3f97..95aa7f43296 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -7,6 +7,7 @@ use clap::{App, Arg, ArgMatches}; use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG; use env_logger::{Builder, Env}; use environment::EnvironmentBuilder; +use eth2_hashing::have_sha_extensions; use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK}; use lighthouse_version::VERSION; use malloc_utils::configure_memory_allocator; @@ -45,10 +46,13 @@ fn main() { .long_version( format!( "{}\n\ - BLS Library: {}\n\ - Specs: mainnet (true), minimal ({}), v0.12.3 ({})", - VERSION.replace("Lighthouse/", ""), bls_library_name(), - cfg!(feature = "spec-minimal"), cfg!(feature = "spec-v12"), + BLS library: {}\n\ + SHA256 hardware acceleration: {}\n\ + Specs: mainnet (true), minimal ({})", + VERSION.replace("Lighthouse/", ""), + bls_library_name(), + have_sha_extensions(), + cfg!(feature = "spec-minimal"), ).as_str() ) .arg( @@ -207,11 +211,7 @@ fn main() { EthSpecId::Mainnet => run(EnvironmentBuilder::mainnet(), &matches, testnet_config), #[cfg(feature = "spec-minimal")] EthSpecId::Minimal => run(EnvironmentBuilder::minimal(), &matches, testnet_config), - #[cfg(feature = "spec-v12")] - EthSpecId::V012Legacy => { - run(EnvironmentBuilder::v012_legacy(), &matches, testnet_config) - } - #[cfg(any(not(feature = "spec-minimal"), not(feature = "spec-v12")))] + #[cfg(not(feature = "spec-minimal"))] other => { eprintln!( "Eth spec `{}` is not supported by this build of Lighthouse",