diff --git a/src/Paprika.Tests/Chain/BlockchainTests.cs b/src/Paprika.Tests/Chain/BlockchainTests.cs index 771dffa7..134decb9 100644 --- a/src/Paprika.Tests/Chain/BlockchainTests.cs +++ b/src/Paprika.Tests/Chain/BlockchainTests.cs @@ -181,7 +181,7 @@ public async Task Account_destruction_same_block() [Category(Categories.LongRunning)] public async Task Account_destruction_spin() { - using var db = PagedDb.NativeMemoryDb(8 * Mb, 2); + using var db = PagedDb.NativeMemoryDb(10 * Mb, 2); await using var blockchain = new Blockchain(db, new ComputeMerkleBehavior()); var parent = Keccak.EmptyTreeHash; diff --git a/src/Paprika.Tests/Chain/RawTests.cs b/src/Paprika.Tests/Chain/RawTests.cs index d7cc4784..0d17b81d 100644 --- a/src/Paprika.Tests/Chain/RawTests.cs +++ b/src/Paprika.Tests/Chain/RawTests.cs @@ -106,7 +106,7 @@ public async Task Disposal() { var account = Values.Key1; - using var db = PagedDb.NativeMemoryDb(256 * 1024, 2); + using var db = PagedDb.NativeMemoryDb(512 * 1024, 2); var merkle = new ComputeMerkleBehavior(); await using var blockchain = new Blockchain(db, merkle); diff --git a/src/Paprika/Store/AbandonedList.cs b/src/Paprika/Store/AbandonedList.cs index 64ae11f1..47b061a2 100644 --- a/src/Paprika/Store/AbandonedList.cs +++ b/src/Paprika/Store/AbandonedList.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Diagnostics.Contracts; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; namespace Paprika.Store; @@ -16,7 +17,7 @@ public struct AbandonedList /// private const int EntriesStart = DbAddress.Size + sizeof(uint); - public const int Size = Page.PageSize - PageHeader.Size - RootPage.Payload.AbandonedStart - EntriesStart; + public const int Size = Page.PageSize; private const int EntrySize = sizeof(uint) + DbAddress.Size; private const int MaxCount = (Size - EntriesStart) / EntrySize; @@ -254,4 +255,7 @@ public bool IsFullyEmpty } public DbAddress GetCurrentForTest() => Current; + + public static ref AbandonedList Wrap(Page page) => + ref Unsafe.As(ref MemoryMarshal.GetReference(page.Span)); } \ No newline at end of file diff --git a/src/Paprika/Store/PagedDb.cs b/src/Paprika/Store/PagedDb.cs index 5a8984fa..6d54f277 100644 --- a/src/Paprika/Store/PagedDb.cs +++ b/src/Paprika/Store/PagedDb.cs @@ -67,7 +67,7 @@ public sealed class PagedDb : IPageResolver, IDb, IDisposable // pooled objects private Context? _ctx; - private readonly BufferPool _pooledRoots; + private readonly BufferPool _pooledPages; #if TRACKING_REUSED_PAGES // reuse tracking @@ -118,7 +118,7 @@ private PagedDb(IPageManager manager, byte historyDepth) "The number of pages registered to be reused"); #endif // Pool - _pooledRoots = new BufferPool(16, true, _meter); + _pooledPages = new BufferPool(16, true, _meter); } public static PagedDb NativeMemoryDb(long size, byte historyDepth = 2) => @@ -193,7 +193,7 @@ public double Megabytes public void Dispose() { - _pooledRoots.Dispose(); + _pooledPages.Dispose(); _manager.Dispose(); _meter.Dispose(); } @@ -225,11 +225,15 @@ public int CountReadOnlyBatches() private ReadOnlyBatch BeginReadOnlyBatch(string name, in RootPage root) { - var copy = new RootPage(_pooledRoots.Rent(false)); + // Root + var rootCopy = new RootPage(_pooledPages.Rent(false)); + root.CopyTo(rootCopy); - root.CopyTo(copy); + // Abandoned + var abandonedCopy = _pooledPages.Rent(false); + GetAt(GetAddress(root.AsPage()).Next).CopyTo(abandonedCopy); - var batch = new ReadOnlyBatch(this, copy, name); + var batch = new ReadOnlyBatch(this, rootCopy, abandonedCopy, name); _batchesReadOnly.Add(batch); return batch; } @@ -344,9 +348,12 @@ public void Accept(IPageVisitor visitor) foreach (var root in _roots) { - using (visitor.On(root, DbAddress.Page(i++))) + var addr = DbAddress.Page(i); + i += DbPagesPerRoot; + + using (visitor.On(root, addr)) { - root.Accept(visitor, this); + root.Accept(visitor, this, GetAt(addr.Next)); } } } @@ -354,10 +361,11 @@ public void Accept(IPageVisitor visitor) public void VisitRoot(IPageVisitor visitor) { var root = Root; + var addr = GetAddress(Root.AsPage()); - using (visitor.On(root, GetAddress(Root.AsPage()))) + using (visitor.On(root, addr)) { - root.Accept(visitor, this); + root.Accept(visitor, this, GetAt(addr.Next)); } } @@ -381,8 +389,10 @@ private void DisposeReadOnlyBatch(ReadOnlyBatch batch) lock (_batchLock) { _batchesReadOnly.Remove(batch); - _pooledRoots.Return(batch.Root.AsPage()); } + + _pooledPages.Return(batch.Root.AsPage()); + _pooledPages.Return(batch.Abandoned.AsPage()); } private IBatch BuildFromRoot(RootPage rootPage) @@ -396,6 +406,10 @@ private IBatch BuildFromRoot(RootPage rootPage) // always inc the batchId root.Header.BatchId++; + // copy abandoned + var abandoned = ctx.PageForAbandoned; + GetAt(GetAddress(rootPage.AsPage()).Next).CopyTo(abandoned); + // copying done above, to minimize the lock lock (_batchLock) { @@ -413,7 +427,7 @@ private IBatch BuildFromRoot(RootPage rootPage) minBatch = Math.Min(batch.BatchId, minBatch); } - return _batchCurrent = new Batch(this, root, minBatch, ctx); + return _batchCurrent = new Batch(this, root, abandoned, minBatch, ctx); } [DoesNotReturn] @@ -435,16 +449,17 @@ static void ThrowOnlyOneBatch() /// private DbAddress SetNewRoot(RootPage root) { - var pageAddress = (_lastRoot + 1) % _historyDepth; - - root.CopyTo(_roots[pageAddress]); - return DbAddress.Page((uint)pageAddress); + var next = NextRootIndex; + root.CopyTo(_roots[next]); + return DbAddress.Page(next * DbPagesPerRoot); } + private uint NextRootIndex => (uint)(_lastRoot + 1) % _historyDepth; + private void CommitNewRoot() => _lastRoot += 1; - private sealed class ReadOnlyBatch(PagedDb db, RootPage root, string name) + private sealed class ReadOnlyBatch(PagedDb db, RootPage root, Page abandoned, string name) : IReportingReadOnlyBatch, IReadOnlyBatchContext { [ThreadStatic] private static ConcurrentDictionary? s_cache; @@ -454,6 +469,7 @@ private sealed class ReadOnlyBatch(PagedDb db, RootPage root, string name) RootPage.IdCacheLimit); public RootPage Root => root; + public Page Abandoned => abandoned; private long _reads; private volatile bool _disposed; @@ -496,8 +512,8 @@ public void Report(IReporter state, IReporter storage, IReporter ids, out long t { ref readonly var data = ref root.Data; - totalAbandoned = 0; - totalAbandoned = data.AbandonedList.GatherTotalAbandoned(this); + + totalAbandoned = AbandonedList.Wrap(abandoned).GatherTotalAbandoned(this); if (data.StateRoot.IsNull == false) { @@ -530,8 +546,9 @@ class Batch : BatchContextBase, IBatch { private readonly PagedDb _db; private readonly RootPage _root; + private readonly Page _abandonedPage; private readonly uint _reusePagesOlderThanBatchId; - private bool _verify = false; + private bool _verify; private bool _disposed; private readonly Context _ctx; @@ -548,11 +565,13 @@ class Batch : BatchContextBase, IBatch private readonly BatchMetrics _metrics; - public Batch(PagedDb db, RootPage root, uint reusePagesOlderThanBatchId, Context ctx) : base( + + public Batch(PagedDb db, RootPage root, Page abandoned, uint reusePagesOlderThanBatchId, Context ctx) : base( root.Header.BatchId) { _db = db; _root = root; + _abandonedPage = abandoned; _reusePagesOlderThanBatchId = reusePagesOlderThanBatchId; _ctx = ctx; _abandoned = ctx.Abandoned; @@ -629,12 +648,12 @@ public async ValueTask Commit(CommitOptions options) CheckDisposed(); // memoize the abandoned so that it's preserved for future uses - MemoizeAbandoned(); + MemoizeAbandoned(_db.NextRootIndex); if (_verify) { using var missing = new MissingPagesVisitor(_root, _db._historyDepth); - _root.Accept(missing, this); + _root.Accept(missing, this, _abandonedPage); missing.EnsureNoMissing(this); } @@ -726,20 +745,27 @@ public override Page GetNewPage(out DbAddress addr, bool clear) return page; } - private void MemoizeAbandoned() + private void MemoizeAbandoned(uint nextRootIndex) { - if (_abandoned.Count == 0) + ref var list = ref AbandonedList.Wrap(_abandonedPage); + + if (_abandoned.Count > 0) { - // nothing to memoize - return; + list.Register(_abandoned, this); } - _root.Data.AbandonedList.Register(_abandoned, this); + var addr = DbAddress.Page(nextRootIndex * DbPagesPerRoot + 1); + + Debug.Assert(addr < _db.NextFreePage, "Abandoned should not breach the range of pages at the beginning of the database"); + Debug.Assert(addr.Raw % DbPagesPerRoot == 1, "Abandoned page should lie next to the root"); + + _abandonedPage.CopyTo(GetAt(addr)); + _written.Add(addr); } private bool TryGetNoLongerUsedPage(out DbAddress found) { - var claimed = _root.Data.AbandonedList.TryGet(out found, _reusePagesOlderThanBatchId, this); + var claimed = AbandonedList.Wrap(_abandonedPage).TryGet(out found, _reusePagesOlderThanBatchId, this); #if TRACKING_REUSED_PAGES if (claimed) @@ -815,6 +841,7 @@ private sealed class Context public unsafe Context() { PageForRoot = new((byte*)NativeMemory.AlignedAlloc(Page.PageSize, (UIntPtr)UIntPtr.Size)); + PageForAbandoned = new((byte*)NativeMemory.AlignedAlloc(Page.PageSize, (UIntPtr)UIntPtr.Size)); Abandoned = new List(); Written = new HashSet(); IdCache = new Dictionary(); @@ -823,6 +850,7 @@ public unsafe Context() public Dictionary IdCache { get; } public Page PageForRoot { get; } + public Page PageForAbandoned { get; } public List Abandoned { get; } public HashSet Written { get; } diff --git a/src/Paprika/Store/RootPage.cs b/src/Paprika/Store/RootPage.cs index d98e5043..d34faec6 100644 --- a/src/Paprika/Store/RootPage.cs +++ b/src/Paprika/Store/RootPage.cs @@ -77,14 +77,6 @@ public struct Payload public FanOutList Ids => new(MemoryMarshal.CreateSpan(ref IdsPayload, FanOutList.FanOut)); - public const int AbandonedStart = - DbAddress.Size * 2 + sizeof(uint) + Metadata.Size + FanOutList.Size * 2; - - /// - /// The start of the abandoned pages. - /// - [FieldOffset(AbandonedStart)] public AbandonedList AbandonedList; - public DbAddress GetNextFreePage() { var free = NextFreePage; @@ -93,7 +85,7 @@ public DbAddress GetNextFreePage() } } - public void Accept(IPageVisitor visitor, IPageResolver resolver) + public void Accept(IPageVisitor visitor, IPageResolver resolver, Page abandoned) { var stateRoot = Data.StateRoot; if (stateRoot.IsNull == false) @@ -103,7 +95,8 @@ public void Accept(IPageVisitor visitor, IPageResolver resolver) Data.Ids.Accept(visitor, resolver); Data.Storage.Accept(visitor, resolver); - Data.AbandonedList.Accept(visitor, resolver); + + AbandonedList.Wrap(abandoned).Accept(visitor, resolver); } ///