fixes for 8k and keeping local

src/Paprika.Tests/Merkle/RootHashFuzzyTests.cs

@@ -74,7 +74,7 @@ public async Task CalculateStateRootHash(
     {
         var generator = Build(test);
 
-        using var db = PagedDb.NativeMemoryDb(16 * 1024 * 1024, 2);
+        using var db = PagedDb.NativeMemoryDb(32 * 1024 * 1024, 2);
         var parallelism = parallel ? ComputeMerkleBehavior.ParallelismUnlimited : ComputeMerkleBehavior.ParallelismNone;
         var merkle = new ComputeMerkleBehavior(parallelism);
 

src/Paprika.Tests/Store/AbandonedTests.cs

@@ -82,14 +82,14 @@ public void Properly_handles_page_addresses_that_are_packed_2()
     private const int HistoryDepth = 2;
 
     [TestCase(20, 1, 10_000, false, TestName = "Accounts - 1")]
-    [TestCase(428, 100, 10_000, false, TestName = "Accounts - 100")]
-    [TestCase(19285, 4000, 200, false,
+    [TestCase(369, 100, 10_000, false, TestName = "Accounts - 100")]
+    [TestCase(12359, 4000, 200, false,
         TestName = "Accounts - 4000 to get a bit reuse",
         Category = Categories.LongRunning)]
-    [TestCase(48240, 10_000, 50, false,
+    [TestCase(31200, 10_000, 50, false,
         TestName = "Accounts - 10000 to breach the AbandonedPage",
         Category = Categories.LongRunning)]
-    [TestCase(88262, 20_000, 50, true,
+    [TestCase(88212, 20_000, 50, true,
         TestName = "Storage - 20_000 accounts with a single storage slot",
         Category = Categories.LongRunning)]
     public async Task Reuse_in_limited_environment(int pageCount, int accounts, int repeats, bool isStorage)

src/Paprika.Tests/Store/PageStructurePrintingTests.cs

@@ -62,16 +62,17 @@ Keccak GetStorageAddress(int i)
         }
     }
 
-    [Test]
-    public async Task Merkle_storage_account()
+    [TestCase(400_000)]
+    [TestCase(300_000)]
+    [TestCase(200_000)]
+    public async Task Merkle_storage_account(int storageSlots)
     {
         var account = Keccak.EmptyTreeHash;
 
         using var db = PagedDb.NativeMemoryDb(MB256);
 
         await using var blockchain = new Blockchain(db, new ComputeMerkleBehavior());
 
-        const int storageSlots = 400_000;
 
         var value = new byte[32];
 

src/Paprika.Tests/Store/PagedDbTests.cs

@@ -10,7 +10,7 @@ namespace Paprika.Tests.Store;
 
 public class PagedDbTests
 {
-    private const int Mb = 1024 * 1024;
+    private const long Mb = 1024 * 1024;
     private const int Seed = 17;
 
     [Test]
@@ -183,7 +183,7 @@ public async Task Multiple_storages_per_commit()
         const int accounts = 512 * 1024;
         const int size = 10_000;
 
-        using var db = PagedDb.NativeMemoryDb(1024 * Mb, 2);
+        using var db = PagedDb.NativeMemoryDb(2 * 1024 * Mb, 2);
 
         var value = new byte[1] { 13 };
 

src/Paprika/Store/DataPage.cs

@@ -169,7 +169,7 @@ private static void Set(DbAddress at, in NibblePath key, in ReadOnlySpan<byte> d
                 }
 
                 // First, try to flush the existing
-                if (TryFindMostFrequentExistingNibble(map, payload.Buckets, out var nibble))
+                if (TryFindMostFrequentExistingNibble(k.Oddity, map, payload.Buckets, out var nibble))
                 {
                     childAddr = EnsureExistingChildWritable(batch, ref payload, nibble);
                     FlushDown(map, nibble, childAddr, batch);
@@ -179,7 +179,7 @@ private static void Set(DbAddress at, in NibblePath key, in ReadOnlySpan<byte> d
                 }
 
                 // None of the existing was flushable, find the most frequent one
-                nibble = FindMostFrequentNibble(map);
+                nibble = FindMostFrequentNibble(k.Oddity, map);
 
                 // Ensure that the child page exists
                 childAddr = payload.Buckets[nibble];
@@ -314,19 +314,24 @@ private static void TurnToFanOut(DbAddress current, in MapSource.Of2 overflow, I
     /// <summary>
     /// A single method for stats gathering to make it simple to change the implementation.
     /// </summary>
-    private static void GatherStats(in SlottedArray map, Span<ushort> stats)
+    private static void GatherStats(int oddity, in SlottedArray map, Span<ushort> stats)
     {
         map.GatherCountStats1Nibble(stats);
 
         // Provide a minor discount for nibbles that should be kept local
         for (byte nibble = 0; nibble < BucketCount; nibble++)
         {
             ref var s = ref stats[nibble];
-            if (s > 0 && ShouldKeepShortKeyLocal(nibble))
+
+            if (s > 0 &&
+                ShouldKeepShortKeyLocal(nibble) &&
+                map.Contains(NibblePath.Single(nibble, oddity)))
             {
+                // The count is bigger than 0 and the nibble should be kept local so test for the key. Only then subtract.
                 s -= 1;
             }
         }
+
         // other proposal to be size based, not count based
         //map.GatherSizeStats1Nibble(stats);
     }
@@ -388,13 +393,13 @@ private static DbAddress EnsureExistingChildWritable(IBatchContext batch, ref Pa
         return childAddr;
     }
 
-    private static byte FindMostFrequentNibble(in SlottedArray map)
+    private static byte FindMostFrequentNibble(int oddity, in SlottedArray map)
     {
         const int count = SlottedArray.BucketCount;
 
         Span<ushort> stats = stackalloc ushort[count];
 
-        GatherStats(map, stats);
+        GatherStats(oddity, map, stats);
 
         byte biggestIndex = 0;
         for (byte i = 1; i < count; i++)
@@ -408,12 +413,12 @@ private static byte FindMostFrequentNibble(in SlottedArray map)
         return biggestIndex;
     }
 
-    private static bool TryFindMostFrequentExistingNibble(in SlottedArray map, in DbAddressList.Of16 children,
+    private static bool TryFindMostFrequentExistingNibble(int oddity, in SlottedArray map, in DbAddressList.Of16 children,
         out byte nibble)
     {
         Span<ushort> stats = stackalloc ushort[BucketCount];
 
-        GatherStats(map, stats);
+        GatherStats(oddity, map, stats);
 
         byte biggestIndex = 0;
         ushort biggestValue = 0;
@@ -470,7 +475,7 @@ private static bool ShouldKeepShortKeyLocal(in NibblePath path) =>
         path.Length == 1 && ShouldKeepShortKeyLocal(path.Nibble0);
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    private static bool ShouldKeepShortKeyLocal(byte nibble) => nibble % 4 == 0;
+    private static bool ShouldKeepShortKeyLocal(byte nibble) => nibble % 2 == 0;
 
     /// <summary>
     /// Represents the data of this data page. This type of payload stores data in 16 nibble-addressable buckets.

src/Paprika/Store/Page.cs

@@ -95,7 +95,7 @@ public struct PageHeader
 /// </summary>
 public readonly unsafe struct Page : IPage, IEquatable<Page>
 {
-    public const int PageSize = 4 * 1024;
+    public const int PageSize = 8 * 1024;
 
     private readonly byte* _ptr;