Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SlottedArray supports 16kb #389

Closed
wants to merge 3 commits into from
Closed

SlottedArray supports 16kb #389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bcbf507
SlottedArray and pages
Scooletz Sep 11, 2024
9136cec
test fixes
Scooletz Sep 11, 2024
7df1b7c
Fixed assertions
Scooletz Sep 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions src/Paprika.Benchmarks/SlottedArrayBenchmarks.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ public unsafe class SlottedArrayBenchmarks
{
private const int KeyCount = 97;

private const int Even = 0;

private const int
BytesPerKey =
3; // 3 repeated bytes allow to cut off the first nibble and still have a unique key. Also, allow storing some key leftover
Expand Down Expand Up @@ -44,7 +46,7 @@ public SlottedArrayBenchmarks()
_map = AllocAlignedPage();
Span<byte> value = stackalloc byte[1];

var map = new SlottedArray(new Span<byte>(_map, Page.PageSize));
var map = new SlottedArray(new Span<byte>(_map, Page.PageSize), Even);
for (byte i = 0; i < KeyCount; i++)
{
value[0] = i;
Expand Down Expand Up @@ -74,7 +76,7 @@ public SlottedArrayBenchmarks()

_hashCollidingMap = AllocAlignedPage();

var hashColliding = new SlottedArray(new Span<byte>(_hashCollidingMap, Page.PageSize));
var hashColliding = new SlottedArray(new Span<byte>(_hashCollidingMap, Page.PageSize), Even);
for (byte i = 0; i < HashCollidingKeyCount; i++)
{
value[0] = i;
Expand Down Expand Up @@ -109,7 +111,7 @@ public SlottedArrayBenchmarks()
[Arguments((byte)KeyCount - 1, false)]
public int TryGet(byte index, bool odd)
{
var map = new SlottedArray(new Span<byte>(_map, Page.PageSize));
var map = new SlottedArray(new Span<byte>(_map, Page.PageSize), Even);
var key = GetKey(index, odd);

var count = 0;
Expand All @@ -129,7 +131,7 @@ public int TryGet(byte index, bool odd)
[Arguments((byte)31)]
public int TryGet_With_Hash_Collisions(byte index)
{
var map = new SlottedArray(new Span<byte>(_hashCollidingMap, Page.PageSize));
var map = new SlottedArray(new Span<byte>(_hashCollidingMap, Page.PageSize), Even);
var key = GetHashCollidingKey(index);

var count = 0;
Expand Down Expand Up @@ -173,7 +175,7 @@ public int Prepare_Key(int sliceFrom, int length)
[Benchmark]
public int EnumerateAll()
{
var map = new SlottedArray(new Span<byte>(_map, Page.PageSize));
var map = new SlottedArray(new Span<byte>(_map, Page.PageSize), Even);

var length = 0;
foreach (var item in map.EnumerateAll())
Expand All @@ -190,7 +192,7 @@ public int EnumerateAll()
[Arguments((byte)1)]
public int EnumerateNibble(byte nibble)
{
var map = new SlottedArray(new Span<byte>(_map, Page.PageSize));
var map = new SlottedArray(new Span<byte>(_map, Page.PageSize), Even);

var length = 0;
foreach (var item in map.EnumerateNibble(nibble))
Expand Down
Binary file modified src/Paprika.Tests/Data/SlottedArrayTests.Defragment_when_no_more_space.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified src/Paprika.Tests/Data/SlottedArrayTests.Enumerate_all_odd=0.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified src/Paprika.Tests/Data/SlottedArrayTests.Enumerate_all_odd=1.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified ...ka.Tests/Data/SlottedArrayTests.Enumerate_long_key_oddStart=0_lengthCutOff=0.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified ...ka.Tests/Data/SlottedArrayTests.Enumerate_long_key_oddStart=0_lengthCutOff=1.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified ...ka.Tests/Data/SlottedArrayTests.Enumerate_long_key_oddStart=1_lengthCutOff=0.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified ...ka.Tests/Data/SlottedArrayTests.Enumerate_long_key_oddStart=1_lengthCutOff=1.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified src/Paprika.Tests/Data/SlottedArrayTests.Enumerate_nibble_nibble=1.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified src/Paprika.Tests/Data/SlottedArrayTests.Enumerate_nibble_nibble=2.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified src/Paprika.Tests/Data/SlottedArrayTests.Enumerate_nibble_nibble=3.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified src/Paprika.Tests/Data/SlottedArrayTests.Enumerate_nibble_nibble=4.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified src/Paprika.Tests/Data/SlottedArrayTests.Set_Get_Delete_Get_AnotherSet.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified src/Paprika.Tests/Data/SlottedArrayTests.Set_Get_Empty.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified src/Paprika.Tests/Data/SlottedArrayTests.Update_in_resize.verified.bin
View file
Open in desktop
Binary file not shown.
Binary file modified src/Paprika.Tests/Data/SlottedArrayTests.Update_in_situ.verified.bin
View file
Open in desktop
Binary file not shown.
139 changes: 72 additions & 67 deletions src/Paprika.Tests/Data/SlottedArrayTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,24 @@ namespace Paprika.Tests.Data;

public class SlottedArrayTests
{
private static ReadOnlySpan<byte> Data0 => new byte[] { 23 };
private static ReadOnlySpan<byte> Data1 => new byte[] { 29, 31 };
private static ReadOnlySpan<byte> Data0 => [23];
private static ReadOnlySpan<byte> Data1 => [29, 31];

private static ReadOnlySpan<byte> Data2 => new byte[] { 37, 39 };
private static ReadOnlySpan<byte> Data2 => [37, 39];

private static ReadOnlySpan<byte> Data3 => new byte[] { 31, 41 };
private static ReadOnlySpan<byte> Data4 => new byte[] { 23, 24, 25 };
private static ReadOnlySpan<byte> Data5 => new byte[] { 23, 24, 64 };
private static ReadOnlySpan<byte> Data3 => [31, 41];
private static ReadOnlySpan<byte> Data4 => [23, 24, 25];
private static ReadOnlySpan<byte> Data5 => [23, 24, 64];

private const byte Even = 0;

[Test]
public Task Set_Get_Delete_Get_AnotherSet()
{
var key0 = Values.Key0.Span;

Span<byte> span = stackalloc byte[SlottedArray.MinimalSizeWithNoData + key0.Length + Data0.Length];
var map = new SlottedArray(span);
var map = new SlottedArray(span, Even);

map.SetAssert(key0, Data0);

Expand All @@ -40,34 +42,47 @@ public Task Set_Get_Delete_Get_AnotherSet()
}

[Test]
public Task Enumerate_all([Values(0, 1)] int odd)
public Task Enumerate_all([Values((byte)0, (byte)1)] byte odd)
{
var isEven = odd == 0;

Span<byte> span = stackalloc byte[256];
var map = new SlottedArray(span);
var map = new SlottedArray(span, odd);

var key0 = NibblePath.Empty;
var key1 = NibblePath.FromKey([7]).SliceFrom(odd);
var key2 = NibblePath.FromKey([7, 13]).SliceFrom(odd);
var key3 = NibblePath.FromKey([7, 13, 31]).SliceFrom(odd);
var key4 = NibblePath.FromKey([7, 13, 31, 41]).SliceFrom(odd);

map.SetAssert(key0, Data0);
if (isEven)
{
map.SetAssert(key0, Data0);
}

map.SetAssert(key1, Data1);
map.SetAssert(key2, Data2);
map.SetAssert(key3, Data3);
map.SetAssert(key4, Data4);

map.GetAssert(key0, Data0);
if (isEven)
{
map.GetAssert(key0, Data0);
}

map.GetAssert(key1, Data1);
map.GetAssert(key2, Data2);
map.GetAssert(key3, Data3);
map.GetAssert(key4, Data4);

using var e = map.EnumerateAll();

e.MoveNext().Should().BeTrue();
e.Current.Key.Equals(key0).Should().BeTrue();
e.Current.RawData.SequenceEqual(Data0).Should().BeTrue();
if (isEven)
{
e.MoveNext().Should().BeTrue();
e.Current.Key.Equals(key0).Should().BeTrue();
e.Current.RawData.SequenceEqual(Data0).Should().BeTrue();
}

e.MoveNext().Should().BeTrue();
e.Current.Key.Equals(key1).Should().BeTrue();
Expand Down Expand Up @@ -95,7 +110,7 @@ public Task Enumerate_all([Values(0, 1)] int odd)
public Task Enumerate_nibble([Values(1, 2, 3, 4)] int nibble)
{
Span<byte> span = stackalloc byte[256];
var map = new SlottedArray(span);
var map = new SlottedArray(span, Even);

var key0 = NibblePath.Empty;
var key1 = NibblePath.FromKey([0x1A]);
Expand Down Expand Up @@ -152,7 +167,7 @@ public void Enumerate_2_nibbles([Values(1, 2, 3, 4)] int nibble0)
const byte nibble1 = 0xA;

Span<byte> span = stackalloc byte[256];
var map = new SlottedArray(span);
var map = new SlottedArray(span, Even);

var key0 = NibblePath.Empty;
var key1 = NibblePath.FromKey([0x10 | nibble1]);
Expand Down Expand Up @@ -201,10 +216,10 @@ public void Enumerate_2_nibbles([Values(1, 2, 3, 4)] int nibble0)
}

[Test]
public Task Enumerate_long_key([Values(0, 1)] int oddStart, [Values(0, 1)] int lengthCutOff)
public Task Enumerate_long_key([Values((byte)0, (byte)1)] byte oddStart, [Values(0, 1)] int lengthCutOff)
{
Span<byte> span = stackalloc byte[128];
var map = new SlottedArray(span);
var map = new SlottedArray(span, oddStart);

var key = NibblePath.FromKey(Keccak.EmptyTreeHash).SliceFrom(oddStart)
.SliceTo(NibblePath.KeccakNibbleCount - oddStart - lengthCutOff);
Expand All @@ -230,7 +245,7 @@ public Task Set_Get_Empty()
var key0 = Values.Key0.Span;

Span<byte> span = stackalloc byte[128];
var map = new SlottedArray(span);
var map = new SlottedArray(span, Even);

var data = ReadOnlySpan<byte>.Empty;

Expand Down Expand Up @@ -261,7 +276,7 @@ public Task Defragment_when_no_more_space()
{
// by trial and error, found the smallest value that will allow to put these two
Span<byte> span = stackalloc byte[SlottedArray.MinimalSizeWithNoData + 88];
var map = new SlottedArray(span);
var map = new SlottedArray(span, Even);

var key0 = Values.Key0.Span;
var key1 = Values.Key1.Span;
Expand Down Expand Up @@ -289,7 +304,7 @@ public Task Update_in_situ()
{
// by trial and error, found the smallest value that will allow to put these two
Span<byte> span = stackalloc byte[128];
var map = new SlottedArray(span);
var map = new SlottedArray(span, Even);

var key1 = Values.Key1.Span;

Expand All @@ -309,7 +324,7 @@ public Task Update_in_resize()

// Update the value, with the next one being bigger.
Span<byte> span = stackalloc byte[SlottedArray.MinimalSizeWithNoData + key0.Length + Data0.Length];
var map = new SlottedArray(span);
var map = new SlottedArray(span, Even);

map.SetAssert(key0, Data0);
map.SetAssert(key0, Data2);
Expand All @@ -324,7 +339,7 @@ public Task Update_in_resize()
public Task Small_keys_compression()
{
Span<byte> span = stackalloc byte[512];
var map = new SlottedArray(span);
var map = new SlottedArray(span, Even);

Span<byte> key = stackalloc byte[1];
Span<byte> value = stackalloc byte[2];
Expand Down Expand Up @@ -353,9 +368,8 @@ public Task Small_keys_compression()
return Verify(span.ToArray());
}

[TestCase(0)]
[TestCase(1)]
public void Key_of_length_5(int odd)
[Test]
public void Key_of_length_5([Values((byte)0, (byte)1)] byte odd)
{
const int length = 5;

Expand All @@ -364,9 +378,9 @@ public void Key_of_length_5(int odd)

Span<byte> span = stackalloc byte[SlottedArray.MinimalSizeWithNoData + spaceForKey];

var key = NibblePath.FromKey(stackalloc byte[] { 0x34, 0x5, 0x7A }, odd, length);
var key = NibblePath.FromKey([0x34, 0x5, 0x7A], odd, length);

var map = new SlottedArray(span);
var map = new SlottedArray(span, odd);

var value = ReadOnlySpan<byte>.Empty;
map.SetAssert(key, value);
Expand All @@ -384,9 +398,9 @@ public void Key_of_length_6_even()
Span<byte> span = stackalloc byte[SlottedArray.MinimalSizeWithNoData + spaceForKey];

// 0b10 is the prefix of the nibble that can be densely encoded on one byte.
var key = NibblePath.FromKey(stackalloc byte[] { 0x34, 0b1001_1101, 0x7A }, 0, length);
var key = NibblePath.FromKey([0x34, 0b1001_1101, 0x7A], 0, length);

var map = new SlottedArray(span);
var map = new SlottedArray(span, Even);

var value = ReadOnlySpan<byte>.Empty;
map.SetAssert(key, value);
Expand All @@ -404,9 +418,9 @@ public void Key_of_length_6_odd()
Span<byte> span = stackalloc byte[SlottedArray.MinimalSizeWithNoData + spaceForKey];

// 0b10 is the prefix of the nibble that can be densely encoded on one byte. For odd, first 3 are consumed to prepare.
var key = NibblePath.FromKey(stackalloc byte[] { 0x04, 0b1011_0010, 0xD9, 0x7A }, 0, length);
var key = NibblePath.FromKey([0x04, 0b1011_0010, 0xD9, 0x7A], 0, length);

var map = new SlottedArray(span);
var map = new SlottedArray(span, Even);

var value = ReadOnlySpan<byte>.Empty;
map.SetAssert(key, value);
Expand All @@ -420,7 +434,7 @@ public void Breach_VectorSize_with_key_count()
var random = new Random(seed);
Span<byte> key = stackalloc byte[4];

var map = new SlottedArray(new byte[3 * 1024]);
var map = new SlottedArray(new byte[3 * 1024], Even);

const int count = 257;

Expand Down Expand Up @@ -451,7 +465,7 @@ public void Set_Get_With_Specific_Lengths([Values(8, 16, 32, 64, 68, 72)] int co
keys[i * keyLength + 1] = i;
}

var map = new SlottedArray(new byte[3 * 1024]);
var map = new SlottedArray(new byte[3 * 1024], Even);

for (var i = 0; i < count; i++)
{
Expand All @@ -476,8 +490,8 @@ public void Set_Get_With_Specific_Lengths([Values(8, 16, 32, 64, 68, 72)] int co
[TestCase(new[] { 0, 1, 7 })]
public void Remove_keys_from(int[] indexes)
{
var toRemove = new SlottedArray(stackalloc byte[512]);
var map = new SlottedArray(stackalloc byte[512]);
var toRemove = new SlottedArray(stackalloc byte[512], Even);
var map = new SlottedArray(stackalloc byte[512], Even);

var key1 = NibblePath.Parse("1");
var key2 = NibblePath.Parse("23");
Expand Down Expand Up @@ -543,8 +557,8 @@ public void Remove_keys_from(int[] indexes)
[Test]
public void Move_to_1()
{
var original = new SlottedArray(stackalloc byte[256]);
var copy0 = new SlottedArray(stackalloc byte[256]);
var original = new SlottedArray(stackalloc byte[256], Even);
var copy0 = new SlottedArray(stackalloc byte[256], Even);

var key1 = NibblePath.Parse("1");
var key2 = NibblePath.Parse("23");
Expand Down Expand Up @@ -585,8 +599,8 @@ public void Move_to_respects_tombstones()
{
const int size = 256;

var original = new SlottedArray(stackalloc byte[size]);
var copy0 = new SlottedArray(stackalloc byte[size]);
var original = new SlottedArray(stackalloc byte[size], Even);
var copy0 = new SlottedArray(stackalloc byte[size], Even);

var key1 = NibblePath.Parse("1");
var key2 = NibblePath.Parse("23");
Expand Down Expand Up @@ -625,9 +639,9 @@ public void Move_to_respects_tombstones()
[Test]
public void Move_to_2()
{
var original = new SlottedArray(stackalloc byte[256]);
var copy0 = new SlottedArray(stackalloc byte[256]);
var copy1 = new SlottedArray(stackalloc byte[256]);
var original = new SlottedArray(stackalloc byte[256], Even);
var copy0 = new SlottedArray(stackalloc byte[256], Even);
var copy1 = new SlottedArray(stackalloc byte[256], Even);

var key0 = NibblePath.Empty;
var key1 = NibblePath.Parse("1");
Expand Down Expand Up @@ -676,11 +690,11 @@ public void Move_to_2()
[Test]
public void Move_to_4()
{
var original = new SlottedArray(stackalloc byte[256]);
var copy0 = new SlottedArray(stackalloc byte[256]);
var copy1 = new SlottedArray(stackalloc byte[256]);
var copy2 = new SlottedArray(stackalloc byte[256]);
var copy3 = new SlottedArray(stackalloc byte[256]);
var original = new SlottedArray(stackalloc byte[256], Even);
var copy0 = new SlottedArray(stackalloc byte[256], Even);
var copy1 = new SlottedArray(stackalloc byte[256], Even);
var copy2 = new SlottedArray(stackalloc byte[256], Even);
var copy3 = new SlottedArray(stackalloc byte[256], Even);

var key0 = NibblePath.Empty;
var key1 = NibblePath.Parse("1");
Expand Down Expand Up @@ -747,15 +761,15 @@ public void Move_to_4()
[Test]
public void Move_to_8()
{
var original = new SlottedArray(stackalloc byte[512]);
var copy0 = new SlottedArray(stackalloc byte[128]);
var copy1 = new SlottedArray(stackalloc byte[128]);
var copy2 = new SlottedArray(stackalloc byte[128]);
var copy3 = new SlottedArray(stackalloc byte[128]);
var copy4 = new SlottedArray(stackalloc byte[128]);
var copy5 = new SlottedArray(stackalloc byte[128]);
var copy6 = new SlottedArray(stackalloc byte[128]);
var copy7 = new SlottedArray(stackalloc byte[128]);
var original = new SlottedArray(stackalloc byte[512], Even);
var copy0 = new SlottedArray(stackalloc byte[128], Even);
var copy1 = new SlottedArray(stackalloc byte[128], Even);
var copy2 = new SlottedArray(stackalloc byte[128], Even);
var copy3 = new SlottedArray(stackalloc byte[128], Even);
var copy4 = new SlottedArray(stackalloc byte[128], Even);
var copy5 = new SlottedArray(stackalloc byte[128], Even);
var copy6 = new SlottedArray(stackalloc byte[128], Even);
var copy7 = new SlottedArray(stackalloc byte[128], Even);

var key0 = NibblePath.Empty;
var key1 = NibblePath.Parse("1");
Expand Down Expand Up @@ -895,16 +909,7 @@ public void Prepare_UnPrepare(int sliceFrom, int length)
{
var key = NibblePath.FromKey(Keccak.EmptyTreeHash).Slice(sliceFrom, length);

// prepare
var hash = SlottedArray.PrepareKeyForTests(key, out var preamble, out var trimmed);
var written = trimmed.IsEmpty ? ReadOnlySpan<byte>.Empty : trimmed.WriteTo(stackalloc byte[33]);

Span<byte> working = stackalloc byte[32];

var unprepared = SlottedArray.UnPrepareKeyForTests(hash, preamble, written, working, out var data);

data.IsEmpty.Should().BeTrue();
key.Equals(unprepared).Should().BeTrue();
SlottedArray.PrepareUnPrepareForTests(key);
}
}

Expand Down
Loading
Loading