Merge pull request #382 from NethermindEth/delete-by-prefix

Delete by prefix delete

src/Paprika.Tests/Chain/RawTests.cs → src/Paprika.Tests/Chain/RawStateTests.cs

@@ -9,7 +9,7 @@
 
 namespace Paprika.Tests.Chain;
 
-public class RawTests
+public class RawStateTests
 {
     [Test]
     public async Task Raw_access_spin()
@@ -126,4 +126,28 @@ public async Task Disposal()
 
         raw.Hash.Should().Be(Keccak.EmptyTreeHash);
     }
+
+    [Test]
+    public async Task DeleteByPrefix()
+    {
+        var account = Values.Key1;
+
+        using var db = PagedDb.NativeMemoryDb(256 * 1024, 2);
+        var merkle = new ComputeMerkleBehavior();
+
+        await using var blockchain = new Blockchain(db, merkle);
+
+        using var raw = blockchain.StartRaw();
+
+        raw.SetAccount(account, new Account(1, 1));
+        raw.Commit();
+
+        raw.RegisterDeleteByPrefix(Key.Account(account));
+        raw.Commit();
+
+        raw.Finalize(1);
+
+        using var read = db.BeginReadOnlyBatch();
+        read.TryGet(Key.Account(account), out _).Should().BeFalse();
+    }
 }

src/Paprika.Tests/Store/PagedDbTests.cs

@@ -90,6 +90,91 @@ byte[] GetData()
         }
     }
 
+    [Test]
+    public async Task DeleteByPrefix_Accounts()
+    {
+        using var db = PagedDb.NativeMemoryDb(16 * Mb, 2);
+
+        var keccak0 = Values.Key0;
+        var keccak1 = Values.Key0;
+        var keccak2 = Values.Key0;
+        var keccak3 = Values.Key0;
+        var prefix = Values.Key0;
+
+        keccak0.BytesAsSpan[^1] = 0x01;
+        keccak1.BytesAsSpan[^1] = 0x02;
+        keccak2.BytesAsSpan[^1] = 0x03;
+        keccak3.BytesAsSpan[^1] = 0x04;
+        prefix.BytesAsSpan[^1] = 0x00;
+
+        // Set data
+        using var batch = db.BeginNextBatch();
+
+        var v = new byte[] { 1 };
+        batch.SetRaw(Key.Account(keccak0), v);
+        batch.SetRaw(Key.Account(keccak1), v);
+        batch.SetRaw(Key.Account(keccak2), v);
+        batch.SetRaw(Key.Account(keccak3), v);
+
+        await batch.Commit(CommitOptions.FlushDataAndRoot);
+
+        // Delete by prefix
+        using var batch2 = db.BeginNextBatch();
+        batch.DeleteByPrefix(Key.Merkle(NibblePath.FromKey(prefix).SliceTo(NibblePath.KeccakNibbleCount - 1)));
+        await batch2.Commit(CommitOptions.FlushDataAndRoot);
+
+        using var read = db.BeginReadOnlyBatch();
+
+        read.TryGet(Key.Account(keccak0), out _).Should().BeFalse();
+        read.TryGet(Key.Account(keccak1), out _).Should().BeFalse();
+        read.TryGet(Key.Account(keccak2), out _).Should().BeFalse();
+        read.TryGet(Key.Account(keccak3), out _).Should().BeFalse();
+    }
+
+    [Test]
+    public async Task DeleteByPrefix_Storage()
+    {
+        using var db = PagedDb.NativeMemoryDb(16 * Mb, 2);
+
+        var account = Values.Key2;
+
+        var keccak0 = Values.Key0;
+        var keccak1 = Values.Key0;
+        var keccak2 = Values.Key0;
+        var keccak3 = Values.Key0;
+        var prefix = Values.Key0;
+
+        keccak0.BytesAsSpan[^1] = 0x01;
+        keccak1.BytesAsSpan[^1] = 0x02;
+        keccak2.BytesAsSpan[^1] = 0x03;
+        keccak3.BytesAsSpan[^1] = 0x04;
+        prefix.BytesAsSpan[^1] = 0x00;
+
+        // Set data
+        using var batch = db.BeginNextBatch();
+
+        var v = new byte[] { 1 };
+        batch.SetRaw(Key.StorageCell(NibblePath.FromKey(account), keccak0), v);
+        batch.SetRaw(Key.StorageCell(NibblePath.FromKey(account), keccak1), v);
+        batch.SetRaw(Key.StorageCell(NibblePath.FromKey(account), keccak2), v);
+        batch.SetRaw(Key.StorageCell(NibblePath.FromKey(account), keccak3), v);
+
+        await batch.Commit(CommitOptions.FlushDataAndRoot);
+
+        // Delete by prefix
+        using var batch2 = db.BeginNextBatch();
+        batch.DeleteByPrefix(Key.Raw(NibblePath.FromKey(account), DataType.Merkle,
+            NibblePath.FromKey(prefix).SliceTo(NibblePath.KeccakNibbleCount - 1)));
+        await batch2.Commit(CommitOptions.FlushDataAndRoot);
+
+        using var read = db.BeginReadOnlyBatch();
+
+        read.TryGet(Key.StorageCell(NibblePath.FromKey(account), keccak0), out _).Should().BeFalse();
+        read.TryGet(Key.StorageCell(NibblePath.FromKey(account), keccak1), out _).Should().BeFalse();
+        read.TryGet(Key.StorageCell(NibblePath.FromKey(account), keccak2), out _).Should().BeFalse();
+        read.TryGet(Key.StorageCell(NibblePath.FromKey(account), keccak3), out _).Should().BeFalse();
+    }
+
     [Test]
     public async Task Multiple_storages_per_commit()
     {

src/Paprika/Chain/Blockchain.cs

@@ -1,3 +1,4 @@
+using System.Buffers;
 using System.CodeDom.Compiler;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
@@ -1632,6 +1633,7 @@ public async ValueTask DisposeAsync()
     /// </summary>
     private class RawState : IRawState
     {
+        private ArrayBufferWriter<byte> _prefixesToDelete = new();
         private readonly Blockchain _blockchain;
         private readonly IDb _db;
         private BlockState _current;
@@ -1698,6 +1700,13 @@ public void SetStorage(in Keccak address, in Keccak storage, ReadOnlySpan<byte>
 
         public void DestroyAccount(in Keccak address) => _current.DestroyAccount(address);
 
+        public void RegisterDeleteByPrefix(in Key prefix)
+        {
+            var span = _prefixesToDelete.GetSpan(prefix.MaxByteLength);
+            var written = prefix.WriteTo(span);
+            _prefixesToDelete.Advance(written.Length);
+        }
+
         public void Commit()
         {
             ThrowOnFinalized();
@@ -1714,6 +1723,8 @@ public void Commit()
 
             using var batch = _db.BeginNextBatch();
 
+            DeleteByPrefixes(batch);
+
             var committed = _current.CommitRaw();
             committed.Apply(batch);
             _current.Dispose();
@@ -1727,6 +1738,17 @@ public void Commit()
             _current = new BlockState(Keccak.Zero, read, ancestors, _blockchain);
         }
 
+        private void DeleteByPrefixes(IBatch batch)
+        {
+            var prefixes = _prefixesToDelete.WrittenSpan;
+            while (prefixes.IsEmpty == false)
+            {
+                prefixes = Key.ReadFrom(prefixes, out var prefixToDelete);
+                batch.DeleteByPrefix(prefixToDelete);
+            }
+            _prefixesToDelete.ResetWrittenCount();
+        }
+
         public void Finalize(uint blockNumber)
         {
             ThrowOnFinalized();

src/Paprika/Chain/IRawState.cs

@@ -17,6 +17,12 @@ public interface IRawState : IReadOnlyWorldState
 
     void DestroyAccount(in Keccak address);
 
+    /// <summary>
+    /// Registers a deletion that will be applied when <see cref="Commit"/> is called.
+    /// </summary>
+    /// <param name="prefix"></param>
+    void RegisterDeleteByPrefix(in Key prefix);
+
     /// <summary>
     /// Commits the pending changes.
     /// </summary>

src/Paprika/Data/Key.cs

@@ -1,9 +1,7 @@
 using System.Diagnostics;
-using System.IO.Hashing;
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using Paprika.Crypto;
-using Paprika.Store;
 
 namespace Paprika.Data;
 
@@ -98,7 +96,6 @@ public static ReadOnlySpan<byte> ReadFrom(ReadOnlySpan<byte> source, out Key key
     public bool IsState => Type == DataType.Account ||
                            (Type == DataType.Merkle && Path.Length < NibblePath.KeccakNibbleCount);
 
-
     [SkipLocalsInit]
     public override int GetHashCode()
     {

src/Paprika/Data/NibblePath.cs

@@ -644,6 +644,15 @@ public override string ToString()
 
     private static readonly char[] Hex = "0123456789ABCDEF".ToArray();
 
+    // TODO: optimize
+    public bool StartsWith(in NibblePath prefix)
+    {
+        if (prefix.Length > Length)
+            return false;
+
+        return SliceTo(prefix.Length).Equals(prefix);
+    }
+
     public bool Equals(in NibblePath other)
     {
         if (((other.Length ^ Length) | (other._odd ^ _odd)) > 0)

src/Paprika/Data/SlottedArray.cs

@@ -83,6 +83,34 @@ public bool TrySet(in NibblePath key, ReadOnlySpan<byte> data)
         return TrySetImpl(hash, preamble, trimmed, data);
     }
 
+    public void DeleteByPrefix(in NibblePath prefix)
+    {
+        if (prefix.Length == 0)
+        {
+            Delete(prefix);
+        }
+        else if (prefix.Length == 1)
+        {
+            // TODO: optimize by filtering by hash. The key is at least 2 nibbles long so can be easily filtered with a bitwise mask over the hash.
+            // Don't materialize data! 
+            foreach (var item in EnumerateNibble(prefix.FirstNibble))
+            {
+                Delete(item);
+            }
+        }
+        else
+        {
+            // TODO: optimize by filtering by hash. The key is at least 2 nibbles long so can be easily filtered with a bitwise mask over the hash.
+            foreach (var item in EnumerateAll())
+            {
+                if (item.Key.StartsWith(prefix))
+                {
+                    Delete(item);
+                }
+            }
+        }
+    }
+
     private bool TrySetImpl(ushort hash, byte preamble, in NibblePath trimmed, ReadOnlySpan<byte> data)
     {
         var index = TryGetImpl(trimmed, hash, preamble, out var existingData);
@@ -955,7 +983,6 @@ public static NibblePath UnPrepareKey(ushort hash, byte preamble, ReadOnlySpan<b
 
             data = input;
 
-
             switch (lengthBits)
             {
                 case KeyPreambleLength0:

src/Paprika/IBatch.cs

@@ -20,6 +20,11 @@ public interface IBatch : IReadOnlyBatch
     /// </summary>
     void Destroy(in NibblePath account);
 
+    /// <summary>
+    /// Deletes all the keys that share the given prefix.
+    /// </summary>
+    void DeleteByPrefix(in Key prefix);
+
     /// <summary>
     /// Commits the block returning its root hash.
     /// </summary>
@@ -33,7 +38,7 @@ public interface IBatch : IReadOnlyBatch
     IBatchStats? Stats { get; }
 
     /// <summary>
-    /// Performs a time consuming verification when <see cref="Commit"/> is called that all the pages are reachable.
+    /// Performs a time-consuming verification when <see cref="Commit"/> is called that all the pages are reachable.
     /// </summary>
     void VerifyDbPagesOnCommit();
 }

src/Paprika/Store/DataPage.cs

@@ -3,7 +3,6 @@
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using Paprika.Data;
-
 using static Paprika.Merkle.Node;
 
 namespace Paprika.Store;
@@ -31,13 +30,39 @@ public readonly unsafe struct DataPage(Page page) : IPageWithData<DataPage>
 
     public ref Payload Data => ref Unsafe.AsRef<Payload>(page.Payload);
 
-    public Page Set(in NibblePath key, in ReadOnlySpan<byte> data, IBatchContext batch)
+    public Page DeleteByPrefix(in NibblePath prefix, IBatchContext batch)
     {
-        if (page.Header.Level > 10)
+        if (Header.BatchId != batch.BatchId)
+        {
+            // the page is from another batch, meaning, it's readonly. Copy
+            var writable = batch.GetWritableCopy(page);
+            return new DataPage(writable).DeleteByPrefix(prefix, batch);
+        }
+
+        Map.DeleteByPrefix(prefix);
+
+        if (prefix.IsEmpty == false)
         {
-            Debugger.Break();
+            var childAddr = Data.Buckets[prefix.FirstNibble];
+
+            if (childAddr.IsNull == false)
+            {
+                var sliced = prefix.SliceFrom(ConsumedNibbles);
+                var child = batch.GetAt(childAddr);
+
+                child = child.Header.PageType == PageType.Leaf
+                    ? new LeafPage(child).DeleteByPrefix(sliced, batch)
+                    : new DataPage(child).DeleteByPrefix(sliced, batch);
+
+                Data.Buckets[prefix.FirstNibble] = batch.GetAddress(child);
+            }
         }
 
+        return page;
+    }
+
+    public Page Set(in NibblePath key, in ReadOnlySpan<byte> data, IBatchContext batch)
+    {
         if (Header.BatchId != batch.BatchId)
         {
             // the page is from another batch, meaning, it's readonly. Copy
@@ -227,8 +252,7 @@ public struct Payload
 
         private const int DataOffset = Size - DataSize;
 
-        [FieldOffset(0)]
-        public DbAddressList.Of16 Buckets;
+        [FieldOffset(0)] public DbAddressList.Of16 Buckets;
 
         /// <summary>
         /// The first item of map of frames to allow ref to it.
@@ -244,7 +268,8 @@ public struct Payload
     public bool TryGet(IReadOnlyBatchContext batch, scoped in NibblePath key, out ReadOnlySpan<byte> result)
         => TryGet(batch, key, out result, this);
 
-    private static bool TryGet(IReadOnlyBatchContext batch, scoped in NibblePath key, out ReadOnlySpan<byte> result, DataPage page)
+    private static bool TryGet(IReadOnlyBatchContext batch, scoped in NibblePath key, out ReadOnlySpan<byte> result,
+        DataPage page)
     {
         var returnValue = false;
         var sliced = key;
@@ -334,4 +359,4 @@ public void Accept(IPageVisitor visitor, IPageResolver resolver, DbAddress addr)
             }
         }
     }
-}
+}

src/Paprika/Store/FanOutListOf256.cs

@@ -54,6 +54,24 @@ public void Set(in NibblePath key, in ReadOnlySpan<byte> data, IBatchContext bat
         _addresses[index] = batch.GetAddress(updated);
     }
 
+    public void DeleteByPrefix(in NibblePath path, IBatchContext batch)
+    {
+        var index = GetIndex(path);
+        var sliced = path.SliceFrom(ConsumedNibbles);
+
+        var addr = _addresses[index];
+
+        if (addr.IsNull)
+        {
+            // There's nothing to delete here
+            return;
+        }
+
+        // The page exists, update
+        var updated = TPage.Wrap(batch.GetAt(addr)).DeleteByPrefix(sliced, batch);
+        _addresses[index] = batch.GetAddress(updated);
+    }
+
     public void Report(IReporter reporter, IPageResolver resolver, int level, int trimmedNibbles)
     {
         var consumedNibbles = trimmedNibbles + ConsumedNibbles;