fix(trie): separate caches for view client and client (#3460)

Currently view client and client share the same cache inside trie. This causes problem when there is some expensive request coming in from view client (state part request for example), which will cause lock contention and can potentially block the client actor. This PR creates separate cache for view client. Notice that while the cache can contain stale data that have been deleted, view client is tolerant to such transient inconsistencies so there should not be an issue.

Test plan
---------
nayduck run http://nayduck.eastus.cloudapp.azure.com:3000/#/run/539

chain/chain/src/test_utils.rs

@@ -270,6 +270,10 @@ impl RuntimeAdapter for KeyValueRuntime {
         self.tries.get_trie_for_shard(shard_id)
     }
 
+    fn get_view_trie_for_shard(&self, shard_id: ShardId) -> Trie {
+        self.tries.get_view_trie_for_shard(shard_id)
+    }
+
     fn verify_block_vrf(
         &self,
         _epoch_id: &EpochId,

chain/chain/src/types.rs

@@ -231,6 +231,9 @@ pub trait RuntimeAdapter: Send + Sync {
     /// Returns trie.
     fn get_trie_for_shard(&self, shard_id: ShardId) -> Trie;
 
+    /// Returns trie with view cache
+    fn get_view_trie_for_shard(&self, shard_id: ShardId) -> Trie;
+
     fn verify_block_vrf(
         &self,
         epoch_id: &EpochId,

core/store/src/trie/shard_tries.rs

@@ -14,29 +14,52 @@ use std::sync::Arc;
 #[derive(Clone)]
 pub struct ShardTries {
     pub(crate) store: Arc<Store>,
+    /// Cache reserved for client actor to use
     pub(crate) caches: Arc<Vec<TrieCache>>,
+    /// Cache for readers.
+    pub(crate) view_caches: Arc<Vec<TrieCache>>,
 }
 
 impl ShardTries {
+    fn get_new_cache(num_shards: NumShards) -> Arc<Vec<TrieCache>> {
+        Arc::new((0..num_shards).map(|_| TrieCache::new()).collect::<Vec<_>>())
+    }
+
     pub fn new(store: Arc<Store>, num_shards: NumShards) -> Self {
         assert_ne!(num_shards, 0);
-        let caches = Arc::new((0..num_shards).map(|_| TrieCache::new()).collect::<Vec<_>>());
-        ShardTries { store, caches }
+        ShardTries {
+            store,
+            caches: Self::get_new_cache(num_shards),
+            view_caches: Self::get_new_cache(num_shards),
+        }
     }
 
     pub fn new_trie_update(&self, shard_id: ShardId, state_root: CryptoHash) -> TrieUpdate {
         TrieUpdate::new(Rc::new(self.get_trie_for_shard(shard_id)), state_root)
     }
 
-    pub fn get_trie_for_shard(&self, shard_id: ShardId) -> Trie {
-        let store = Box::new(TrieCachingStorage::new(
-            self.store.clone(),
-            self.caches[shard_id as usize].clone(),
-            shard_id,
-        ));
+    pub fn new_trie_update_view(&self, shard_id: ShardId, state_root: CryptoHash) -> TrieUpdate {
+        TrieUpdate::new(Rc::new(self.get_view_trie_for_shard(shard_id)), state_root)
+    }
+
+    fn get_trie_for_shard_internal(&self, shard_id: ShardId, is_view: bool) -> Trie {
+        let cache = if is_view {
+            self.view_caches[shard_id as usize].clone()
+        } else {
+            self.caches[shard_id as usize].clone()
+        };
+        let store = Box::new(TrieCachingStorage::new(self.store.clone(), cache, shard_id));
         Trie::new(store, shard_id)
     }
 
+    pub fn get_trie_for_shard(&self, shard_id: ShardId) -> Trie {
+        self.get_trie_for_shard_internal(shard_id, false)
+    }
+
+    pub fn get_view_trie_for_shard(&self, shard_id: ShardId) -> Trie {
+        self.get_trie_for_shard_internal(shard_id, true)
+    }
+
     pub fn get_store(&self) -> Arc<Store> {
         self.store.clone()
     }

neard/src/runtime.rs

@@ -501,6 +501,10 @@ impl RuntimeAdapter for NightshadeRuntime {
         self.tries.get_trie_for_shard(shard_id)
     }
 
+    fn get_view_trie_for_shard(&self, shard_id: ShardId) -> Trie {
+        self.tries.get_view_trie_for_shard(shard_id)
+    }
+
     fn verify_block_vrf(
         &self,
         epoch_id: &EpochId,
@@ -1181,7 +1185,7 @@ impl RuntimeAdapter for NightshadeRuntime {
         num_parts: u64,
     ) -> Result<Vec<u8>, Error> {
         assert!(part_id < num_parts);
-        let trie = self.get_trie_for_shard(shard_id);
+        let trie = self.get_view_trie_for_shard(shard_id);
         let result = match trie.get_trie_nodes_for_part(part_id, num_parts, state_root) {
             Ok(partial_state) => partial_state,
             Err(e) => {
@@ -1247,7 +1251,7 @@ impl RuntimeAdapter for NightshadeRuntime {
         shard_id: ShardId,
         state_root: &StateRoot,
     ) -> Result<StateRootNode, Error> {
-        self.get_trie_for_shard(shard_id)
+        self.get_view_trie_for_shard(shard_id)
             .retrieve_root_node(state_root)
             .map_err(|e| e.to_string().into())
     }
@@ -1294,7 +1298,7 @@ impl node_runtime::adapter::ViewRuntimeAdapter for NightshadeRuntime {
         state_root: MerkleHash,
         account_id: &AccountId,
     ) -> Result<Account, Box<dyn std::error::Error>> {
-        let state_update = self.get_tries().new_trie_update(shard_id, state_root);
+        let state_update = self.get_tries().new_trie_update_view(shard_id, state_root);
         self.trie_viewer.view_account(&state_update, account_id)
     }
 
@@ -1314,7 +1318,7 @@ impl node_runtime::adapter::ViewRuntimeAdapter for NightshadeRuntime {
         epoch_info_provider: &dyn EpochInfoProvider,
         current_protocol_version: ProtocolVersion,
     ) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
-        let state_update = self.get_tries().new_trie_update(shard_id, state_root);
+        let state_update = self.get_tries().new_trie_update_view(shard_id, state_root);
         self.trie_viewer.call_function(
             state_update,
             height,
@@ -1338,7 +1342,7 @@ impl node_runtime::adapter::ViewRuntimeAdapter for NightshadeRuntime {
         account_id: &AccountId,
         public_key: &PublicKey,
     ) -> Result<AccessKey, Box<dyn std::error::Error>> {
-        let state_update = self.get_tries().new_trie_update(shard_id, state_root);
+        let state_update = self.get_tries().new_trie_update_view(shard_id, state_root);
         self.trie_viewer.view_access_key(&state_update, account_id, public_key)
     }
 
@@ -1348,7 +1352,7 @@ impl node_runtime::adapter::ViewRuntimeAdapter for NightshadeRuntime {
         state_root: MerkleHash,
         account_id: &AccountId,
     ) -> Result<Vec<(PublicKey, AccessKey)>, Box<dyn std::error::Error>> {
-        let state_update = self.get_tries().new_trie_update(shard_id, state_root);
+        let state_update = self.get_tries().new_trie_update_view(shard_id, state_root);
         let prefix = trie_key_parsers::get_raw_prefix_for_access_keys(account_id);
         let raw_prefix: &[u8] = prefix.as_ref();
         let access_keys = match state_update.iter(&prefix) {
@@ -1375,7 +1379,7 @@ impl node_runtime::adapter::ViewRuntimeAdapter for NightshadeRuntime {
         account_id: &AccountId,
         prefix: &[u8],
     ) -> Result<ViewStateResult, Box<dyn std::error::Error>> {
-        let state_update = self.get_tries().new_trie_update(shard_id, state_root);
+        let state_update = self.get_tries().new_trie_update_view(shard_id, state_root);
         self.trie_viewer.view_state(&state_update, account_id, prefix)
     }
 }