Paprika analysis of storage layout (#390)

src/Paprika.Tests/Store/PageStructurePrintingTests.cs

@@ -1,6 +1,8 @@
 using System.Buffers.Binary;
 using NUnit.Framework;
+using Paprika.Chain;
 using Paprika.Crypto;
+using Paprika.Merkle;
 using Paprika.Store;
 using Spectre.Console;
 
@@ -44,7 +46,52 @@ public async Task Uniform_buckets_spin()
             await batch.Commit(CommitOptions.FlushDataAndRoot);
         }
 
-        var view = new TreeView();
+        var view = new TreeView(db);
+        db.VisitRoot(view);
+
+        AnsiConsole.WriteLine($"DB size: {db.Megabytes:N0}MB");
+        AnsiConsole.Write(view.Tree);
+
+        return;
+
+        Keccak GetStorageAddress(int i)
+        {
+            Keccak result = default;
+            BinaryPrimitives.WriteInt32LittleEndian(result.BytesAsSpan, i);
+            return result;
+        }
+    }
+
+    [Test]
+    public async Task Merkle_storage_account()
+    {
+        var account = Keccak.EmptyTreeHash;
+
+        using var db = PagedDb.NativeMemoryDb(MB256);
+
+        await using var blockchain = new Blockchain(db, new ComputeMerkleBehavior());
+
+        const int storageSlots = 400_000;
+
+        var value = new byte[32];
+
+        var random = new Random(13);
+        random.NextBytes(value);
+
+        using var block = blockchain.StartNew(Keccak.EmptyTreeHash);
+
+        block.SetAccount(account, new Account(1_000, 1_000, Keccak.OfAnEmptyString, Keccak.OfAnEmptySequenceRlp));
+
+        for (var slot = 0; slot < storageSlots; slot++)
+        {
+            block.SetStorage(account, GetStorageAddress(slot), value);
+        }
+
+        var commit = block.Commit(1);
+        blockchain.Finalize(commit);
+        await blockchain.WaitTillFlush(1);
+
+        var view = new TreeView(db);
         db.VisitRoot(view);
 
         AnsiConsole.WriteLine($"DB size: {db.Megabytes:N0}MB");

src/Paprika.Tests/Store/TreeView.cs

@@ -4,26 +4,15 @@
 
 namespace Paprika.Tests.Store;
 
-public class TreeView : IPageVisitor, IDisposable
+public class TreeView(IPageResolver resolver) : IPageVisitor, IDisposable
 {
     public readonly Tree Tree = new("Db");
 
     private readonly Stack<TreeNode> _nodes = new();
 
-    private IDisposable Build(string name, DbAddress? addr, int? capacityLeft = null)
+    private IDisposable BuildNode(string name)
     {
-        string text;
-        if (addr.HasValue)
-        {
-            var capacity = capacityLeft.HasValue ? $", space_left: {capacityLeft.Value}" : "";
-            text = $"{name.Replace("Page", "")}, @{addr.Value.Raw}{capacity}";
-        }
-        else
-        {
-            text = name;
-        }
-
-        var node = new TreeNode(new Text(text));
+        var node = new TreeNode(new Text(name));
 
         if (_nodes.TryPeek(out var parent))
         {
@@ -39,12 +28,36 @@ private IDisposable Build(string name, DbAddress? addr, int? capacityLeft = null
     }
 
     public IDisposable On<TPage>(scoped ref NibblePath.Builder prefix, TPage page, DbAddress addr)
-        where TPage : unmanaged, IPage => Build(page.GetType().Name, addr);
+        where TPage : unmanaged, IPage =>
+        On(page, addr);
+
+    public IDisposable On<TPage>(TPage page, DbAddress addr) where TPage : unmanaged, IPage
+    {
+        var name = page.GetType().Name;
+        var text = $"{name.Replace("Page", "")}, @{addr.Raw}, lvl: {page.AsPage().Header.Level}";
+
+        var p = page.AsPage();
 
-    public IDisposable On<TPage>(TPage page, DbAddress addr) where TPage : unmanaged, IPage =>
-        Build(page.GetType().Name, addr);
+        if (typeof(TPage) == typeof(DataPage))
+        {
+            text += ReportUsage(p.Cast<DataPage>().Map);
+        }
+        else if (typeof(TPage) == typeof(LeafOverflowPage))
+        {
+            text += ReportUsage(p.Cast<LeafOverflowPage>().Map);
+        }
+
+        return BuildNode(text);
+
+        string ReportUsage(SlottedArray map)
+        {
+            var usage = map.CalculateActualSpaceUsed();
+            var report = $", Usage: {usage:P}";
+            return report;
+        }
+    }
 
-    public IDisposable Scope(string name) => Build(name, null);
+    public IDisposable Scope(string name) => BuildNode(name);
 
     public void Dispose() => _nodes.TryPop(out _);
 }

src/Paprika/Data/SlottedArray.cs

@@ -1,5 +1,6 @@
 using System.Buffers.Binary;
 using System.Diagnostics;
+using System.Drawing;
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -189,6 +190,26 @@ private bool TrySetImpl(ushort hash, byte preamble, in NibblePath trimmed, ReadO
     /// </summary>
     public int Count => _header.Low / Slot.TotalSize;
 
+    /// <summary>
+    /// Performs a walk through the map calculating the actual size of data stored in it.
+    /// </summary>
+    public double CalculateActualSpaceUsed()
+    {
+        var occupied = 0;
+
+        foreach (var item in EnumerateAll())
+        {
+            occupied += Slot.TotalSize;
+            occupied += item.RawData.Length;
+            if (item.Key.Length > Slot.KeyPreambleMaxEncodedLength)
+            {
+                occupied += KeyEncoding.GetBytesCount(item.Key.SliceFrom(Slot.KeyPreambleMaxEncodedLength));
+            }
+        }
+
+        return (double)occupied / _data.Length;
+    }
+
     public int CapacityLeft => _data.Length - _header.Taken;
 
     public Enumerator EnumerateAll() => new(this);
@@ -1095,7 +1116,7 @@ public void MarkAsDeleted()
         public const byte KeyPreambleWithBytes = KeyPreambleLength5OrMore << KeyPreambleLengthShift;
 
         private const byte KeyPreambleLengthShift = 1;
-        private const byte KeyPreambleMaxEncodedLength = 4;
+        public const byte KeyPreambleMaxEncodedLength = 4;
         private const byte KeySlice = 2;
 
         private const int HashByteShift = 8;