Tree reporting

src/Paprika.Runner/Program.cs

@@ -1,21 +1,22 @@
 using System.Buffers.Binary;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using HdrHistogram;
 using Nethermind.Int256;
 using Paprika.Chain;
 using Paprika.Crypto;
 using Paprika.Store;
 using Paprika.Tests;
 using Spectre.Console;
+using Spectre.Console.Rendering;
 
 [assembly: ExcludeFromCodeCoverage]
 
 namespace Paprika.Runner;
 
 public static class Program
 {
-    private const int BlockCount = PersistentDb ? 25_000 : 3_000;
-    private const int RandomSampleSize = 260_000_000;
+    private const int BlockCount = PersistentDb ? 20_000 : 3_000;
     private const int AccountsPerBlock = 1000;
     private const int MaxReorgDepth = 64;
     private const int FinalizeEvery = 32;
@@ -27,7 +28,7 @@ public static class Program
     private const long DbFileSize = PersistentDb ? 256 * Gb : 16 * Gb;
     private const long Gb = 1024 * 1024 * 1024L;
 
-    private static readonly TimeSpan FlushEvery = TimeSpan.FromSeconds(5);
+    private static readonly TimeSpan FlushEvery = TimeSpan.FromSeconds(30);
 
     private const int LogEvery = BlockCount / NumberOfLogs;
 
@@ -131,7 +132,7 @@ public static async Task Main(String[] args)
             }
 
             // waiting for finalization
-            var read = db.BeginReadOnlyBatch();
+            using var read = db.BeginReadOnlyBatch();
 
             var readingStopWatch = Stopwatch.StartNew();
             random = BuildRandom();
@@ -184,6 +185,29 @@ public static async Task Main(String[] args)
             // the final report
             ReportReading(counter);
 
+            var stats = new StatisticsReporter();
+            read.Report(stats);
+            var table = new Table();
+
+            table.AddColumn(new TableColumn("Level of Paprika tree"));
+            table.AddColumn(new TableColumn("Child page count"));
+            table.AddColumn(new TableColumn("Entries in page"));
+
+            foreach (var (key, level) in stats.Levels)
+            {
+                table.AddRow(
+                    new Text(key.ToString()),
+                    WriteHistogram(level.ChildCount),
+                    WriteHistogram(level.Entries));
+            }
+
+            var mb = (long)stats.PageCount * Page.PageSize / 1024 / 1024;
+            var report = new Layout().SplitRows(
+                new Layout(new Paragraph($"General stats:\n1. Size of this Paprika tree: {mb}MB")).Size(3),
+                new Layout(table.Expand()));
+
+            layout[info].Update(new Panel(report).Header("Paprika tree statistics").Expand());
+
             spectre.Cancel();
             await reportingTask;
 
@@ -220,6 +244,17 @@ void ReportReading(int i)
 
     private static Random BuildRandom() => new(RandomSeed);
 
+    private static IRenderable WriteHistogram(HistogramBase histogram)
+    {
+        string Percentile(int percentile, string color)
+        {
+            var value = histogram.GetValueAtPercentile(percentile);
+            return $"[{color}]P{percentile}: {value,2}[/] ";
+        }
+
+        return new Markup(Percentile(50, "green") + Percentile(90, "yellow") + Percentile(95, "red"));
+    }
+
     private static int Writer(Blockchain blockchain, Keccak bigStorageAccount, Random random,
         Layout reporting)
     {

src/Paprika/Chain/Blockchain.cs

@@ -278,6 +278,9 @@ public bool TryGet(in Key key, out ReadOnlySpan<byte> result)
                 Dispose();
             }
         }
+
+        public void Report(IReporter reporter) =>
+            throw new NotImplementedException("One should not report over a block");
     }
 
     /// <summary>

src/Paprika/Data/HashingMap.cs

@@ -84,6 +84,15 @@ public bool TryGet(uint hash, in Key key, out ReadOnlySpan<byte> value)
         return false;
     }
 
+    public int Count
+    {
+        get
+        {
+            var indexOf = _hashes.IndexOf(NoHash);
+            return indexOf == IndexOfNotFound ? _hashes.Length : indexOf;
+        }
+    }
+
     private Span<byte> GetEntry(int position) => _entries.Slice(position * EntrySize, EntrySize);
 
     public bool TrySet(uint hash, in Key key, ReadOnlySpan<byte> value)

src/Paprika/IReadOnlyBatch.cs

@@ -1,6 +1,4 @@
-using Nethermind.Int256;
-using Paprika.Crypto;
-using Paprika.Data;
+using Paprika.Data;
 using Paprika.Store;
 
 namespace Paprika;
@@ -16,4 +14,6 @@ public interface IReadOnlyBatch : IDisposable
     /// <param name="result"></param>
     /// <returns></returns>
     bool TryGet(in Key key, out ReadOnlySpan<byte> result);
+
+    void Report(IReporter reporter);
 }

src/Paprika/Paprika.csproj

@@ -8,6 +8,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+      <PackageReference Include="HdrHistogram" Version="2.5.0" />
       <PackageReference Include="Nethermind.Numerics.Int256" Version="1.1.0" />
       <PackageReference Include="System.IO.Hashing" Version="8.0.0-preview.4.23259.5" />
     </ItemGroup>

src/Paprika/Store/DataPage.cs

@@ -38,7 +38,7 @@ public struct Payload
     {
         private const int Size = Page.PageSize - PageHeader.Size;
 
-        private const int BucketCount = 16;
+        public const int BucketCount = 16;
 
         /// <summary>
         /// The size of the raw byte data held in this page. Must be long aligned.
@@ -248,6 +248,34 @@ public bool TryGet(uint hash, Key key, IReadOnlyBatchContext batch, out ReadOnly
         return false;
     }
 
+    public void Report(IReporter reporter, IPageResolver resolver, int level)
+    {
+        var emptyBuckets = 0;
+
+        foreach (var bucket in Data.Buckets)
+        {
+            if (bucket.IsNull)
+            {
+                emptyBuckets++;
+            }
+            else
+            {
+                new DataPage(resolver.GetAt(bucket)).Report(reporter, resolver, level + 1);
+            }
+        }
+
+        if (emptyBuckets == 0)
+        {
+            // all filled
+            reporter.Report(level, Payload.BucketCount, new HashingMap(Data.DataSpan).Count);
+        }
+        else
+        {
+            reporter.Report(level, Payload.BucketCount - emptyBuckets,
+                new NibbleBasedMap(Data.DataSpan).Count);
+        }
+    }
+
     private static bool TryFindExistingStorageTreeForCellOf(in NibbleBasedMap map, in Key key,
         out DbAddress storageTreeAddress)
     {

src/Paprika/Store/IReporter.cs

@@ -0,0 +1,36 @@
+using HdrHistogram;
+
+namespace Paprika.Store;
+
+/// <summary>
+/// Provides capability to report stats for <see cref="DataPage"/>.
+/// </summary>
+public interface IReporter
+{
+    void Report(int level, int filledBuckets, int entriesPerPage);
+}
+
+public class StatisticsReporter : IReporter
+{
+    public readonly SortedDictionary<int, Level> Levels = new();
+    public int PageCount = 0;
+
+    public void Report(int level, int filledBuckets, int entriesPerPage)
+    {
+        if (Levels.TryGetValue(level, out var lvl) == false)
+        {
+            lvl = Levels[level] = new Level();
+        }
+
+        PageCount++;
+
+        lvl.ChildCount.RecordValue(filledBuckets);
+        lvl.Entries.RecordValue(entriesPerPage);
+    }
+
+    public class Level
+    {
+        public readonly IntHistogram ChildCount = new(1000, 5);
+        public readonly IntHistogram Entries = new(1000, 5);
+    }
+}

src/Paprika/Store/Page.cs

@@ -87,7 +87,7 @@ public enum PageType : byte
 /// </summary>
 public readonly unsafe struct Page : IPage, IEquatable<Page>
 {
-    public const int PageCount = 0x0100_0000; // 64GB addressable
+    public const int PageCount = 0x1000_0000; // 64GB addressable
     public const int PageAddressMask = PageCount - 1;
     public const int PageSize = 4 * 1024;
 

src/Paprika/Store/PagedDb.cs

@@ -270,6 +270,17 @@ public bool TryGet(in Key key, out ReadOnlySpan<byte> result)
             return new DataPage(GetAt(addr)).TryGet(hash, sliced, this, out result);
         }
 
+        public void Report(IReporter reporter)
+        {
+            foreach (var addr in _rootDataPages)
+            {
+                if (addr.IsNull == false)
+                {
+                    new DataPage(GetAt(addr)).Report(reporter, this, 1);
+                }
+            }
+        }
+
         public uint BatchId { get; }
 
         public Page GetAt(DbAddress address) => _db._manager.GetAt(address);
@@ -388,6 +399,17 @@ private void CheckDisposed()
             }
         }
 
+        public void Report(IReporter reporter)
+        {
+            foreach (var addr in _root.Data.AccountPages)
+            {
+                if (addr.IsNull == false)
+                {
+                    new DataPage(GetAt(addr)).Report(reporter, this, 1);
+                }
+            }
+        }
+
         public async ValueTask Commit(CommitOptions options)
         {
             var watch = Stopwatch.StartNew();