Find frozen storage slots correctly, refactor the DB segment names

Signed-off-by: Matthew Whitehead <[email protected]>

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java

@@ -36,14 +36,14 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier {
   ACCOUNT_STORAGE_STORAGE(new byte[] {8}, EnumSet.of(BONSAI, BONSAI_ARCHIVE), false, true, false),
   TRIE_BRANCH_STORAGE(new byte[] {9}, EnumSet.of(BONSAI, BONSAI_ARCHIVE), false, true, false),
   TRIE_LOG_STORAGE(new byte[] {10}, EnumSet.of(BONSAI, BONSAI_ARCHIVE), true, false, true),
-  ACCOUNT_FREEZER_STATE(
-      "ACCOUNT_FREEZER_STATE".getBytes(StandardCharsets.UTF_8),
+  ACCOUNT_INFO_STATE_FREEZER(
+      "ACCOUNT_INFO_STATE_FREEZER".getBytes(StandardCharsets.UTF_8),
       EnumSet.of(BONSAI_ARCHIVE),
       true,
       false,
       true),
-  STORAGE_FREEZER_STATE(
-      "STORAGE_FREEZER_STATE".getBytes(StandardCharsets.UTF_8),
+  ACCOUNT_STORAGE_FREEZER(
+      "ACCOUNT_STORAGE_FREEZER".getBytes(StandardCharsets.UTF_8),
       EnumSet.of(BONSAI_ARCHIVE),
       true,
       false,

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/storage/flat/ArchiveFlatDbStrategy.java

@@ -14,8 +14,9 @@
  */
 package org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat;
 
-import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_FREEZER_STATE;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE;
+import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE_FREEZER;
+import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_FREEZER;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE;
 
 import org.hyperledger.besu.datatypes.Hash;
@@ -86,7 +87,7 @@ public Optional<Bytes> getFlatAccount(
       // Check the frozen state as old state is moved out of the primary DB segment
       final Optional<Bytes> frozenAccountFound =
           storage
-              .getNearestBefore(ACCOUNT_FREEZER_STATE, keyNearest)
+              .getNearestBefore(ACCOUNT_INFO_STATE_FREEZER, keyNearest)
               // return empty when we find a "deleted value key"
               .filter(
                   found ->
@@ -167,10 +168,30 @@ public Optional<Bytes> getFlatStorageValueByStorageSlotKey(
 
     if (storageFound.isPresent()) {
       getStorageValueFlatDatabaseCounter.inc();
+      return storageFound;
     } else {
-      getStorageValueNotFoundInFlatDatabaseCounter.inc();
+      // Check the frozen storage as old state is moved out of the primary DB segment
+      final Optional<Bytes> frozenStorageFound =
+          storage
+              .getNearestBefore(ACCOUNT_STORAGE_FREEZER, keyNearest)
+              // return empty when we find a "deleted value key"
+              .filter(
+                  found ->
+                      !Arrays.areEqual(
+                          DELETED_STORAGE_VALUE, found.value().orElse(DELETED_STORAGE_VALUE)))
+              // don't return accounts that do not have a matching account hash
+              .filter(found -> accountHash.commonPrefixLength(found.key()) >= accountHash.size())
+              .flatMap(SegmentedKeyValueStorage.NearestKeyValue::wrapBytes);
+
+      if (frozenStorageFound.isPresent()) {
+        // TODO - different metric for frozen lookups?
+        getStorageValueFlatDatabaseCounter.inc();
+      } else {
+        getStorageValueNotFoundInFlatDatabaseCounter.inc();
+      }
+
+      return frozenStorageFound;
     }
-    return storageFound;
   }
 
   /*

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/bonsai/worldview/BonsaiArchiveFreezer.java

@@ -90,7 +90,7 @@ public synchronized int moveBlockStateToFreezer() {
     }
 
     AtomicInteger frozenAccountStateCount = new AtomicInteger();
-    AtomicInteger frozenStorageStateCount = new AtomicInteger();
+    AtomicInteger frozenAccountStorageCount = new AtomicInteger();
 
     LOG.atDebug()
         .setMessage(
@@ -105,7 +105,8 @@ public synchronized int moveBlockStateToFreezer() {
             .dropWhile((e) -> e.getKey() > retainAboveThisBlock);
     // TODO - limit to a configurable number of blocks to move per loop
 
-    final Multimap<Long, Hash> movedToFreezer = ArrayListMultimap.create();
+    final Multimap<Long, Hash> accountStateFreezerActionsComplete = ArrayListMultimap.create();
+    final Multimap<Long, Hash> accountStorageFreezerActionsComplete = ArrayListMultimap.create();
 
     // Determine which world state keys have changed in the last N blocks by looking at the
     // trie logs for the blocks. Then move the old keys to the freezer segment (if and only if they
@@ -130,7 +131,7 @@ public synchronized int moveBlockStateToFreezer() {
                                     address.addressHash()));
                           });
                 }
-                movedToFreezer.put(block.getKey(), blockHash);
+                accountStateFreezerActionsComplete.put(block.getKey(), blockHash);
               }
             });
 
@@ -153,7 +154,7 @@ public synchronized int moveBlockStateToFreezer() {
                             storageSlotKey.forEach(
                                 (slotKey, slotValue) -> {
                                   // Move any previous state for this account
-                                  frozenStorageStateCount.addAndGet(
+                                  frozenAccountStorageCount.addAndGet(
                                       rootWorldStateStorage.freezePreviousStorageState(
                                           blockchain.getBlockHeader(
                                               blockchain
@@ -165,22 +166,33 @@ public synchronized int moveBlockStateToFreezer() {
                                 });
                           });
                 }
-                movedToFreezer.put(block.getKey(), blockHash);
+                accountStorageFreezerActionsComplete.put(block.getKey(), blockHash);
               }
             });
 
-    movedToFreezer.keySet().forEach(blocksToMoveToFreezer::removeAll);
+    // For us to consider all state and storage changes for a block complete, it must have been
+    // recorded in both accountState and accountStorage lists. If only one finished we need to try
+    // freezing state/storage for that block again on the next loop
+    int frozenBlocksCompleted = blocksToMoveToFreezer.size();
+    accountStateFreezerActionsComplete
+        .keySet()
+        .forEach(
+            (b) -> {
+              if (accountStorageFreezerActionsComplete.containsKey(b)) {
+                blocksToMoveToFreezer.removeAll(b);
+              }
+            });
 
-    if (frozenAccountStateCount.get() > 0 || frozenStorageStateCount.get() > 0) {
+    if (frozenAccountStateCount.get() > 0 || frozenAccountStorageCount.get() > 0) {
       LOG.atInfo()
-          .setMessage("froze {} account state entries, {} storage state entries for {} blocks")
+          .setMessage("froze {} account state entries, {} account storage entries for {} blocks")
           .addArgument(frozenAccountStateCount.get())
-          .addArgument(frozenStorageStateCount.get())
-          .addArgument(movedToFreezer::size)
+          .addArgument(frozenAccountStorageCount.get())
+          .addArgument(frozenBlocksCompleted)
           .log();
     }
 
-    return movedToFreezer.size();
+    return frozenBlocksCompleted;
   }
 
   @Override

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/diffbased/common/storage/DiffBasedWorldStateKeyValueStorage.java

@@ -14,11 +14,11 @@
  */
 package org.hyperledger.besu.ethereum.trie.diffbased.common.storage;
 
-import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_FREEZER_STATE;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE;
+import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE_FREEZER;
+import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_FREEZER;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE;
-import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.STORAGE_FREEZER_STATE;
 import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;
 import static org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.flat.ArchiveFlatDbStrategy.DELETED_ACCOUNT_VALUE;
 
@@ -201,9 +201,9 @@ public boolean pruneTrieLog(final Hash blockHash) {
   }
 
   /**
-   * Move old account state from the primary DB segments to "cold" segments that will only be used
-   * for historic state queries. This prevents performance degradation over time for writes to the
-   * primary DB segments.
+   * Move old account state from the primary DB segments to "freezer" segments that will only be
+   * used for historic state queries. This prevents performance degradation over time for writes to
+   * the primary DB segments.
    *
    * @param previousBlockHeader the block header for the previous block, used to get the "nearest
    *     before" state
@@ -238,7 +238,7 @@ public int freezePreviousAccountState(
                       composedWorldStateStorage.startTransaction();
                   tx.remove(ACCOUNT_INFO_STATE, nearestKey.key().toArrayUnsafe());
                   tx.put(
-                      ACCOUNT_FREEZER_STATE,
+                      ACCOUNT_INFO_STATE_FREEZER,
                       nearestKey.key().toArrayUnsafe(),
                       nearestKey.value().get());
                   tx.commit();
@@ -296,7 +296,7 @@ public int freezePreviousStorageState(
                       composedWorldStateStorage.startTransaction();
                   tx.remove(ACCOUNT_STORAGE_STORAGE, nearestKey.key().toArrayUnsafe());
                   tx.put(
-                      STORAGE_FREEZER_STATE,
+                      ACCOUNT_STORAGE_FREEZER,
                       nearestKey.key().toArrayUnsafe(),
                       nearestKey.value().get());
                   tx.commit();