From 88b0f2af0941f837e56b79c46b35fff320b80dfd Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 22 May 2024 14:38:09 +0300 Subject: [PATCH] add generalised tests to file interface --- share/namespace.go | 4 +- share/shwap/namespaced_data_test.go | 2 +- share/store/file/mem_file_test.go | 55 +++---- share/store/file/testing.go | 217 ++++++++++++++++++++++++++++ 4 files changed, 237 insertions(+), 41 deletions(-) create mode 100644 share/store/file/testing.go diff --git a/share/namespace.go b/share/namespace.go index 3e4e9a98ca..2cea574bbc 100644 --- a/share/namespace.go +++ b/share/namespace.go @@ -185,9 +185,9 @@ func (n Namespace) IsGreaterOrEqualThan(target Namespace) bool { return bytes.Compare(n, target) > -1 } -// Add adds arbitrary int value to namespace, treating namespace as big-endian +// AddInt adds arbitrary int value to namespace, treating namespace as big-endian // implementation of int -func (n Namespace) Add(val int) (Namespace, error) { +func (n Namespace) AddInt(val int) (Namespace, error) { if val == 0 { return n, nil } diff --git a/share/shwap/namespaced_data_test.go b/share/shwap/namespaced_data_test.go index f75d8c1761..b5765763d5 100644 --- a/share/shwap/namespaced_data_test.go +++ b/share/shwap/namespaced_data_test.go @@ -39,7 +39,7 @@ func TestNamespacedRowFromSharesNonIncluded(t *testing.T) { const odsSize = 8 // Test absent namespace shares := sharetest.RandShares(t, odsSize) - absentNs, err := share.GetNamespace(shares[0]).Add(1) + absentNs, err := share.GetNamespace(shares[0]).AddInt(1) require.NoError(t, err) parity, err := share.DefaultRSMT2DCodec().Encode(shares) diff --git a/share/store/file/mem_file_test.go b/share/store/file/mem_file_test.go index 5dd6c87998..f02c02a883 100644 --- a/share/store/file/mem_file_test.go +++ b/share/store/file/mem_file_test.go @@ -1,51 +1,30 @@ package file import ( - "context" - mrand "math/rand" "testing" - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/celestiaorg/rsmt2d" ) -func TestMemFileShare(t *testing.T) { - eds := edstest.RandEDS(t, 32) - root, err := share.NewRoot(eds) - require.NoError(t, err) - fl := &MemFile{Eds: eds} - - width := int(eds.Width()) - for rowIdx := 0; rowIdx < width; rowIdx++ { - for colIdx := 0; colIdx < width; colIdx++ { - shr, err := fl.Share(context.TODO(), rowIdx, colIdx) - require.NoError(t, err) - - ok := shr.VerifyInclusion(root, rowIdx, colIdx) - require.True(t, ok) - } +func TestMemFile(t *testing.T) { + size := 8 + newFile := func(eds *rsmt2d.ExtendedDataSquare) EdsFile { + return &MemFile{Eds: eds} } -} -func TestMemFileDate(t *testing.T) { - size := 32 + t.Run("Share", func(t *testing.T) { + testFileShare(t, newFile, size) + }) - // generate EDS with random data and some shares with the same namespace - namespace := sharetest.RandV0Namespace() - amount := mrand.Intn(size*size-1) + 1 - eds, dah := edstest.RandEDSWithNamespace(t, namespace, amount, size) + t.Run("AxisHalf", func(t *testing.T) { + testFileAxisHalf(t, newFile, size) + }) - file := &MemFile{Eds: eds} + t.Run("Data", func(t *testing.T) { + testFileData(t, newFile, size) + }) - for i, root := range dah.RowRoots { - if !namespace.IsOutsideRange(root, root) { - nd, err := file.Data(context.Background(), namespace, i) - require.NoError(t, err) - ok := nd.VerifyInclusion(root, namespace) - require.True(t, ok) - } - } + t.Run("EDS", func(t *testing.T) { + testFileEds(t, newFile, size) + }) } diff --git a/share/store/file/testing.go b/share/store/file/testing.go new file mode 100644 index 0000000000..05cc3ff787 --- /dev/null +++ b/share/store/file/testing.go @@ -0,0 +1,217 @@ +package file + +import ( + "context" + "fmt" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/stretchr/testify/require" + mrand "math/rand" + "strconv" + "sync" + "testing" + + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" +) + +type createFile func(eds *rsmt2d.ExtendedDataSquare) EdsFile + +func testFileShare(t *testing.T, createFile createFile, odsSize int) { + eds := edstest.RandEDS(t, odsSize) + fl := createFile(eds) + + dah, err := share.NewRoot(eds) + require.NoError(t, err) + + width := int(eds.Width()) + t.Run("single thread", func(t *testing.T) { + for x := 0; x < width; x++ { + for y := 0; y < width; y++ { + testShare(t, fl, eds, dah, x, y) + } + } + }) + + t.Run("parallel", func(t *testing.T) { + wg := sync.WaitGroup{} + for y := 0; y < width; y++ { + for x := 0; x < width; x++ { + wg.Add(1) + go func(x, y int) { + defer wg.Done() + testShare(t, fl, eds, dah, x, y) + }(x, y) + } + } + wg.Wait() + }) +} + +func testShare(t *testing.T, + fl EdsFile, + eds *rsmt2d.ExtendedDataSquare, + dah *share.Root, + x, y int) { + shr, err := fl.Share(context.TODO(), x, y) + require.NoError(t, err) + + ok := shr.VerifyInclusion(dah, x, y) + require.True(t, ok) +} + +func testFileData(t *testing.T, createFile createFile, size int) { + t.Run("included", func(t *testing.T) { + // generate EDS with random data and some Shares with the same namespace + namespace := sharetest.RandV0Namespace() + amount := mrand.Intn(size*size-1) + 1 + eds, dah := edstest.RandEDSWithNamespace(t, namespace, amount, size) + f := createFile(eds) + testData(t, f, namespace, dah) + }) + + t.Run("not included", func(t *testing.T) { + // generate EDS with random data and some Shares with the same namespace + eds := edstest.RandEDS(t, size) + dah, err := share.NewRoot(eds) + require.NoError(t, err) + + maxNs := nmt.MaxNamespace(dah.RowRoots[(len(dah.RowRoots))/2-1], share.NamespaceSize) + targetNs, err := share.Namespace(maxNs).AddInt(-1) + require.NoError(t, err) + + f := createFile(eds) + testData(t, f, targetNs, dah) + }) +} + +func testData(t *testing.T, f EdsFile, namespace share.Namespace, dah *share.Root) { + for i, root := range dah.RowRoots { + if !namespace.IsOutsideRange(root, root) { + nd, err := f.Data(context.Background(), namespace, i) + require.NoError(t, err) + ok := nd.VerifyInclusion(root, namespace) + require.True(t, ok) + } + } +} + +func testFileAxisHalf(t *testing.T, createFile createFile, odsSize int) { + eds := edstest.RandEDS(t, odsSize) + fl := createFile(eds) + + t.Run("single thread", func(t *testing.T) { + for _, axisType := range []rsmt2d.Axis{rsmt2d.Col, rsmt2d.Row} { + for i := 0; i < int(eds.Width()); i++ { + half, err := fl.AxisHalf(context.Background(), axisType, i) + require.NoError(t, err) + require.Len(t, half.Shares, odsSize) + + var expected []share.Share + if half.IsParity { + expected = getAxis(eds, axisType, i)[odsSize:] + } else { + expected = getAxis(eds, axisType, i)[:odsSize] + } + + require.Equal(t, expected, half.Shares) + } + } + }) + + t.Run("parallel", func(t *testing.T) { + wg := sync.WaitGroup{} + for _, axisType := range []rsmt2d.Axis{rsmt2d.Col, rsmt2d.Row} { + for i := 0; i < int(eds.Width()); i++ { + wg.Add(1) + go func(axisType rsmt2d.Axis, idx int) { + defer wg.Done() + half, err := fl.AxisHalf(context.Background(), axisType, idx) + require.NoError(t, err) + require.Len(t, half.Shares, odsSize) + + var expected []share.Share + if half.IsParity { + expected = getAxis(eds, axisType, idx)[odsSize:] + } else { + expected = getAxis(eds, axisType, idx)[:odsSize] + } + + require.Equal(t, expected, half.Shares) + }(axisType, i) + } + } + wg.Wait() + }) +} + +func testFileEds(t *testing.T, createFile createFile, size int) { + eds := edstest.RandEDS(t, size) + fl := createFile(eds) + + eds2, err := fl.EDS(context.Background()) + require.NoError(t, err) + require.True(t, eds.Equals(eds2)) +} + +func benchGetAxisFromFile(b *testing.B, newFile func(size int) EdsFile, minSize, maxSize int) { + for size := minSize; size <= maxSize; size *= 2 { + f := newFile(size) + + // loop over all possible axis types and quadrants + for _, axisType := range []rsmt2d.Axis{rsmt2d.Row, rsmt2d.Col} { + for _, squareHalf := range []int{0, 1} { + name := fmt.Sprintf("Size:%v/ProofType:%s/squareHalf:%s", size, axisType, strconv.Itoa(squareHalf)) + b.Run(name, func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := f.AxisHalf(context.TODO(), axisType, f.Size()/2*(squareHalf)) + require.NoError(b, err) + } + }) + } + } + } +} + +func benchGetShareFromFile(b *testing.B, newFile func(size int) EdsFile, minSize, maxSize int) { + for size := minSize; size <= maxSize; size *= 2 { + f := newFile(size) + + // loop over all possible axis types and quadrants + for _, q := range quadrants { + name := fmt.Sprintf("Size:%v/quadrant:%s", size, q) + b.Run(name, func(b *testing.B) { + x, y := q.coordinates(f.Size()) + // warm up cache + _, err := f.Share(context.TODO(), x, y) + require.NoError(b, err) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := f.Share(context.TODO(), x, y) + require.NoError(b, err) + } + }) + } + + } +} + +type quadrant int + +var ( + quadrants = []quadrant{1, 2, 3, 4} +) + +func (q quadrant) String() string { + return strconv.Itoa(int(q)) +} + +func (q quadrant) coordinates(edsSize int) (x, y int) { + x = edsSize/2*(int(q-1)%2) + 1 + y = edsSize/2*(int(q-1)/2) + 1 + return +}