In-memory tree states (#5533)

* Consensus changes

* EF tests

* lcli

* common and watch

* account manager

* cargo

* fork choice

* promise cache

* beacon chain

* interop genesis

* http api

* lighthouse

* op pool

* beacon chain misc

* parallel state cache

* store

* fix issues in store

* IT COMPILES

* Remove some unnecessary module qualification

* Revert Arced pubkey optimization (#5536)

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Fix caching, rebasing and some tests

* Remove unused deps

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Small cleanups

* Revert shuffling cache/promise cache changes

* Fix state advance bugs

* Fix shuffling tests

* Remove some resolved FIXMEs

* Remove StateProcessingStrategy

* Optimise withdrawals calculation

* Don't reorg if state cache is missed

* Remove inconsistent state func

* Fix beta compiler

* Rebase early, rebase often

* Fix state caching behaviour

* Update to milhouse release

* Fix on-disk consensus context format

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Squashed commit of the following:

commit 3a16649
Author: Michael Sproul <[email protected]>
Date:   Thu Apr 18 14:26:09 2024 +1000

    Fix on-disk consensus context format

* Keep indexed attestations, thanks Sean

* Merge branch 'on-disk-consensus-context' into tree-states-memory

* Merge branch 'unstable' into tree-states-memory

* Address half of Sean's review

* More simplifications from Sean's review

* Cache state after get_advanced_hot_state

Cargo.toml

@@ -114,7 +114,7 @@ discv5 = { version = "0.4.1", features = ["libp2p"] }
 env_logger = "0.9"
 error-chain = "0.12"
 ethereum-types = "0.14"
-ethereum_hashing = "1.0.0-beta.2"
+ethereum_hashing = "0.6.0"
 ethereum_serde_utils = "0.5.2"
 ethereum_ssz = "0.5"
 ethereum_ssz_derive = "0.5"
@@ -132,6 +132,7 @@ libsecp256k1 = "0.7"
 log = "0.4"
 lru = "0.12"
 maplit = "1"
+milhouse = "0.1"
 num_cpus = "1"
 parking_lot = "0.12"
 paste = "1"
@@ -144,6 +145,7 @@ rayon = "1.7"
 regex = "1"
 reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "stream", "rustls-tls", "native-tls-vendored"] }
 ring = "0.16"
+rpds = "0.11"
 rusqlite = { version = "0.28", features = ["bundled"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
@@ -156,9 +158,9 @@ slog-term = "2"
 sloggers = { version = "2", features = ["json"] }
 smallvec = "1.11.2"
 snap = "1"
-ssz_types = "0.5"
+ssz_types = "0.6"
 strum = { version = "0.24", features = ["derive"] }
-superstruct = "0.6"
+superstruct = "0.7"
 syn = "1"
 sysinfo = "0.26"
 tempfile = "3"
@@ -170,8 +172,8 @@ tracing-appender = "0.2"
 tracing-core = "0.1"
 tracing-log = "0.2"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-tree_hash = "0.5"
-tree_hash_derive = "0.5"
+tree_hash = "0.6"
+tree_hash_derive = "0.6"
 url = "2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
 warp = { version = "0.3.7", default-features = false, features = ["tls"] }

beacon_node/beacon_chain/Cargo.toml

@@ -19,58 +19,58 @@ environment = { workspace = true }
 serde_json = { workspace = true }
 
 [dependencies]
-serde_json = { workspace = true }
+bitvec = { workspace = true }
+bls = { workspace = true }
+derivative = { workspace = true }
+eth1 = { workspace = true }
+eth2 = { workspace = true }
 eth2_network_config = { workspace = true }
-merkle_proof = { workspace = true }
-store = { workspace = true }
-parking_lot = { workspace = true }
+ethereum_hashing = { workspace = true }
+ethereum_serde_utils = { workspace = true }
+ethereum_ssz = { workspace = true }
+ethereum_ssz_derive = { workspace = true }
+execution_layer = { workspace = true }
+fork_choice = { workspace = true }
+futures = { workspace = true }
+genesis = { workspace = true }
+hex = { workspace = true }
+int_to_bytes = { workspace = true }
+itertools = { workspace = true }
+kzg = { workspace = true }
 lazy_static = { workspace = true }
-smallvec = { workspace = true }
 lighthouse_metrics = { workspace = true }
+logging = { workspace = true }
+lru = { workspace = true }
+merkle_proof = { workspace = true }
+oneshot_broadcast = { path = "../../common/oneshot_broadcast/" }
 operation_pool = { workspace = true }
+parking_lot = { workspace = true }
+proto_array = { workspace = true }
+rand = { workspace = true }
 rayon = { workspace = true }
+safe_arith = { workspace = true }
+sensitive_url = { workspace = true }
 serde = { workspace = true }
-ethereum_serde_utils = { workspace = true }
+serde_json = { workspace = true }
+slasher = { workspace = true }
 slog = { workspace = true }
+slog-async = { workspace = true }
+slog-term = { workspace = true }
 sloggers = { workspace = true }
 slot_clock = { workspace = true }
-ethereum_hashing = { workspace = true }
-ethereum_ssz = { workspace = true }
+smallvec = { workspace = true }
 ssz_types = { workspace = true }
-ethereum_ssz_derive = { workspace = true }
 state_processing = { workspace = true }
-tree_hash_derive = { workspace = true }
-tree_hash = { workspace = true }
-types = { workspace = true }
-tokio = { workspace = true }
-tokio-stream = { workspace = true }
-eth1 = { workspace = true }
-futures = { workspace = true }
-genesis = { workspace = true }
-int_to_bytes = { workspace = true }
-rand = { workspace = true }
-proto_array = { workspace = true }
-lru = { workspace = true }
-tempfile = { workspace = true }
-bitvec = { workspace = true }
-bls = { workspace = true }
-kzg = { workspace = true }
-safe_arith = { workspace = true }
-fork_choice = { workspace = true }
-task_executor = { workspace = true }
-derivative = { workspace = true }
-itertools = { workspace = true }
-slasher = { workspace = true }
-eth2 = { workspace = true }
+store = { workspace = true }
 strum = { workspace = true }
-logging = { workspace = true }
-execution_layer = { workspace = true }
-sensitive_url = { workspace = true }
 superstruct = { workspace = true }
-hex = { workspace = true }
-oneshot_broadcast = { path = "../../common/oneshot_broadcast/" }
-slog-term = { workspace = true }
-slog-async = { workspace = true }
+task_executor = { workspace = true }
+tempfile = { workspace = true }
+tokio = { workspace = true }
+tokio-stream = { workspace = true }
+tree_hash = { workspace = true }
+tree_hash_derive = { workspace = true }
+types = { workspace = true }
 
 [[test]]
 name = "beacon_chain_tests"