diff --git a/api/docgen/examples.go b/api/docgen/examples.go index 83d25da6df..a8ffc5dfda 100644 --- a/api/docgen/examples.go +++ b/api/docgen/examples.go @@ -64,7 +64,7 @@ var ExampleValues = map[reflect.Type]interface{}{ &byzantine.ErrByzantine{ Index: 0, Axis: rsmt2d.Axis(0), - Shares: []*byzantine.ShareWithProof{}, + Shares: []*share.ShareWithProof{}, }, ), reflect.TypeOf((*error)(nil)).Elem(): errors.New("error"), diff --git a/api/gateway/share_test.go b/api/gateway/share_test.go index 9b12240f62..fe260b8e1b 100644 --- a/api/gateway/share_test.go +++ b/api/gateway/share_test.go @@ -10,7 +10,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" - "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" ) func Test_dataFromShares(t *testing.T) { diff --git a/cmd/node.go b/cmd/node.go index 51ac4a6d2e..e8891c78f5 100644 --- a/cmd/node.go +++ b/cmd/node.go @@ -9,6 +9,7 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + "github.com/celestiaorg/celestia-node/nodebuilder/pruner" "github.com/celestiaorg/celestia-node/nodebuilder/rpc" "github.com/celestiaorg/celestia-node/nodebuilder/state" ) @@ -22,6 +23,7 @@ func NewBridge(options ...func(*cobra.Command, []*pflag.FlagSet)) *cobra.Command rpc.Flags(), gateway.Flags(), state.Flags(), + pruner.Flags(), } cmd := &cobra.Command{ Use: "bridge [subcommand]", @@ -72,6 +74,7 @@ func NewFull(options ...func(*cobra.Command, []*pflag.FlagSet)) *cobra.Command { rpc.Flags(), gateway.Flags(), state.Flags(), + pruner.Flags(), } cmd := &cobra.Command{ Use: "full [subcommand]", diff --git a/cmd/util.go b/cmd/util.go index 08fa02155b..bbc901e4f2 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -16,6 +16,7 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + "github.com/celestiaorg/celestia-node/nodebuilder/pruner" rpc_cfg "github.com/celestiaorg/celestia-node/nodebuilder/rpc" "github.com/celestiaorg/celestia-node/nodebuilder/state" "github.com/celestiaorg/celestia-node/share" @@ -105,13 +106,6 @@ func PersistentPreRunEnv(cmd *cobra.Command, nodeType node.Type, _ []string) err return err } - if nodeType != node.Bridge { - err = header.ParseFlags(cmd, &cfg.Header) - if err != nil { - return err - } - } - ctx, err = ParseMiscFlags(ctx, cmd) if err != nil { return err @@ -121,6 +115,24 @@ func PersistentPreRunEnv(cmd *cobra.Command, nodeType node.Type, _ []string) err gateway.ParseFlags(cmd, &cfg.Gateway) state.ParseFlags(cmd, &cfg.State) + switch nodeType { + case node.Light: + err = header.ParseFlags(cmd, &cfg.Header) + if err != nil { + return err + } + case node.Full: + err = header.ParseFlags(cmd, &cfg.Header) + if err != nil { + return err + } + pruner.ParseFlags(cmd, &cfg.Pruner) + case node.Bridge: + pruner.ParseFlags(cmd, &cfg.Pruner) + default: + panic(fmt.Sprintf("invalid node type: %v", nodeType)) + } + // set config ctx = WithNodeConfig(ctx, &cfg) cmd.SetContext(ctx) diff --git a/core/eds.go b/core/eds.go index eb93c249ba..a6fb8cb883 100644 --- a/core/eds.go +++ b/core/eds.go @@ -1,11 +1,8 @@ package core import ( - "context" - "errors" "fmt" - "github.com/filecoin-project/dagstore" "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/app" @@ -15,9 +12,6 @@ import ( "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" ) // extendBlock extends the given block data, returning the resulting @@ -49,16 +43,3 @@ func extendShares(s [][]byte, options ...nmt.Option) (*rsmt2d.ExtendedDataSquare wrapper.NewConstructor(uint64(squareSize), options...)) } - -// storeEDS will only store extended block if it is not empty and doesn't already exist. -func storeEDS(ctx context.Context, hash share.DataHash, eds *rsmt2d.ExtendedDataSquare, store *eds.Store) error { - if eds == nil { - return nil - } - err := store.Put(ctx, hash, eds) - if errors.Is(err, dagstore.ErrShardExists) { - // block with given root already exists, return nil - return nil - } - return err -} diff --git a/core/exchange.go b/core/exchange.go index cf889a38bb..69f9845909 100644 --- a/core/exchange.go +++ b/core/exchange.go @@ -9,18 +9,17 @@ import ( "golang.org/x/sync/errgroup" libhead "github.com/celestiaorg/go-header" - "github.com/celestiaorg/nmt" "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/libs/utils" + "github.com/celestiaorg/celestia-node/share/store" ) const concurrencyLimit = 4 type Exchange struct { fetcher *BlockFetcher - store *eds.Store + store *store.Store construct header.ConstructFn metrics *exchangeMetrics @@ -28,7 +27,7 @@ type Exchange struct { func NewExchange( fetcher *BlockFetcher, - store *eds.Store, + store *store.Store, construct header.ConstructFn, opts ...Option, ) (*Exchange, error) { @@ -132,10 +131,7 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende } // extend block data - adder := ipld.NewProofsAdder(int(block.Data.SquareSize)) - defer adder.Purge() - - eds, err := extendBlock(block.Data, block.Header.Version.App, nmt.NodeVisitor(adder.VisitFn())) + eds, err := extendBlock(block.Data, block.Header.Version.App) if err != nil { return nil, fmt.Errorf("extending block data for height %d: %w", &block.Height, err) } @@ -150,11 +146,11 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende &block.Height, hash, eh.Hash()) } - ctx = ipld.CtxWithProofsAdder(ctx, adder) - err = storeEDS(ctx, eh.DAH.Hash(), eds, ce.store) + f, err := ce.store.Put(ctx, eh.DAH.Hash(), eh.Height(), eds) if err != nil { return nil, fmt.Errorf("storing EDS to eds.Store for height %d: %w", &block.Height, err) } + utils.CloseAndLog(log, "file", f) return eh, nil } @@ -176,11 +172,7 @@ func (ce *Exchange) getExtendedHeaderByHeight(ctx context.Context, height *int64 } log.Debugw("fetched signed block from core", "height", b.Header.Height) - // extend block data - adder := ipld.NewProofsAdder(int(b.Data.SquareSize)) - defer adder.Purge() - - eds, err := extendBlock(b.Data, b.Header.Version.App, nmt.NodeVisitor(adder.VisitFn())) + eds, err := extendBlock(b.Data, b.Header.Version.App) if err != nil { return nil, fmt.Errorf("extending block data for height %d: %w", b.Header.Height, err) } @@ -190,10 +182,10 @@ func (ce *Exchange) getExtendedHeaderByHeight(ctx context.Context, height *int64 panic(fmt.Errorf("constructing extended header for height %d: %w", b.Header.Height, err)) } - ctx = ipld.CtxWithProofsAdder(ctx, adder) - err = storeEDS(ctx, eh.DAH.Hash(), eds, ce.store) + f, err := ce.store.Put(ctx, eh.DAH.Hash(), eh.Height(), eds) if err != nil { return nil, fmt.Errorf("storing EDS to eds.Store for block height %d: %w", b.Header.Height, err) } + utils.CloseAndLog(log, "file", f) return eh, nil } diff --git a/core/exchange_test.go b/core/exchange_test.go index 95c7f83385..e3076b6cb3 100644 --- a/core/exchange_test.go +++ b/core/exchange_test.go @@ -5,15 +5,13 @@ import ( "testing" "time" - ds "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/test/util/testnode" "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/store" ) func TestCoreExchange_RequestHeaders(t *testing.T) { @@ -62,11 +60,11 @@ func createCoreFetcher(t *testing.T, cfg *testnode.Config) (*BlockFetcher, testn return NewBlockFetcher(cctx.Client), cctx } -func createStore(t *testing.T) *eds.Store { +func createStore(t *testing.T) *store.Store { t.Helper() - storeCfg := eds.DefaultParameters() - store, err := eds.NewStore(storeCfg, t.TempDir(), ds_sync.MutexWrap(ds.NewMapDatastore())) + storeCfg := store.DefaultParameters() + store, err := store.NewStore(storeCfg, t.TempDir()) require.NoError(t, err) return store } diff --git a/core/listener.go b/core/listener.go index 367aa34181..d3274f3e67 100644 --- a/core/listener.go +++ b/core/listener.go @@ -12,12 +12,11 @@ import ( "go.opentelemetry.io/otel/attribute" libhead "github.com/celestiaorg/go-header" - "github.com/celestiaorg/nmt" "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" + "github.com/celestiaorg/celestia-node/share/store" ) var ( @@ -38,7 +37,7 @@ type Listener struct { fetcher *BlockFetcher construct header.ConstructFn - store *eds.Store + store *store.Store headerBroadcaster libhead.Broadcaster[*header.ExtendedHeader] hashBroadcaster shrexsub.BroadcastFn @@ -56,7 +55,7 @@ func NewListener( fetcher *BlockFetcher, hashBroadcaster shrexsub.BroadcastFn, construct header.ConstructFn, - store *eds.Store, + store *store.Store, blocktime time.Duration, opts ...Option, ) (*Listener, error) { @@ -206,11 +205,8 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS span.SetAttributes( attribute.Int64("height", b.Header.Height), ) - // extend block data - adder := ipld.NewProofsAdder(int(b.Data.SquareSize)) - defer adder.Purge() - eds, err := extendBlock(b.Data, b.Header.Version.App, nmt.NodeVisitor(adder.VisitFn())) + eds, err := extendBlock(b.Data, b.Header.Version.App) if err != nil { return fmt.Errorf("extending block data: %w", err) } @@ -222,11 +218,11 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS } // attempt to store block data if not empty - ctx = ipld.CtxWithProofsAdder(ctx, adder) - err = storeEDS(ctx, b.Header.DataHash.Bytes(), eds, cl.store) + f, err := cl.store.Put(ctx, eh.DAH.Hash(), eh.Height(), eds) if err != nil { return fmt.Errorf("storing EDS: %w", err) } + utils.CloseAndLog(log, "file", f) syncing, err := cl.fetcher.IsSyncing(ctx) if err != nil { diff --git a/core/listener_test.go b/core/listener_test.go index b3ed11e571..8789311355 100644 --- a/core/listener_test.go +++ b/core/listener_test.go @@ -17,6 +17,7 @@ import ( nodep2p "github.com/celestiaorg/celestia-node/nodebuilder/p2p" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" + "github.com/celestiaorg/celestia-node/share/store" ) const networkID = "private" @@ -84,11 +85,8 @@ func TestListenerWithWrongChainRPC(t *testing.T) { eds := createEdsPubSub(ctx, t) store := createStore(t) - err := store.Start(ctx) - require.NoError(t, err) t.Cleanup(func() { - err = store.Stop(ctx) - require.NoError(t, err) + require.NoError(t, store.Close()) }) // create Listener and start listening @@ -141,7 +139,7 @@ func createListener( fetcher *BlockFetcher, ps *pubsub.PubSub, edsSub *shrexsub.PubSub, - store *eds.Store, + store *store.Store, chainID string, ) *Listener { p2pSub, err := p2p.NewSubscriber[*header.ExtendedHeader](ps, header.MsgID, p2p.WithSubscriberNetworkID(networkID)) diff --git a/go.mod b/go.mod index ad968ffa1c..b9a4902f18 100644 --- a/go.mod +++ b/go.mod @@ -37,10 +37,8 @@ require ( github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger4 v0.1.5 - github.com/ipfs/go-ipld-cbor v0.1.0 github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipld/go-car v0.6.2 github.com/libp2p/go-libp2p v0.32.2 github.com/libp2p/go-libp2p-kad-dht v0.25.2 github.com/libp2p/go-libp2p-pubsub v0.10.0 @@ -78,6 +76,8 @@ require ( google.golang.org/protobuf v1.32.0 ) +require github.com/ipfs/go-ipld-cbor v0.1.0 // indirect + require ( cloud.google.com/go v0.112.0 // indirect cloud.google.com/go/compute v1.23.3 // indirect @@ -213,7 +213,6 @@ require ( github.com/ipfs/go-ipfs-util v0.0.3 // indirect github.com/ipfs/go-ipld-legacy v0.2.1 // indirect github.com/ipfs/go-log v1.0.5 // indirect - github.com/ipfs/go-merkledag v0.11.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect @@ -228,7 +227,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect - github.com/klauspost/reedsolomon v1.11.8 // indirect + github.com/klauspost/reedsolomon v1.12.1-0.20240110152930-bb8917fa442f github.com/koron/go-ssdp v0.0.4 // indirect github.com/lib/pq v1.10.7 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect @@ -315,7 +314,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 // indirect - go.uber.org/atomic v1.11.0 // indirect + go.uber.org/atomic v1.11.0 go.uber.org/dig v1.17.1 // indirect go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect diff --git a/go.sum b/go.sum index d8f0551653..c7fb1636e0 100644 --- a/go.sum +++ b/go.sum @@ -1065,7 +1065,6 @@ github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMi github.com/ipfs/go-bitswap v0.3.4/go.mod h1:4T7fvNv/LmOys+21tnLzGKncMeeXUYUd1nUiJ2teMvI= github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= github.com/ipfs/go-bitswap v0.6.0/go.mod h1:Hj3ZXdOC5wBJvENtdqsixmzzRukqd8EHLxZLZc3mzRA= -github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= github.com/ipfs/go-bitswap v0.11.0/go.mod h1:05aE8H3XOU+LXpTedeAS0OZpcO1WFsj5niYQH9a1Tmk= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -1173,7 +1172,6 @@ github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3 github.com/ipfs/go-ipfs-redirects-file v0.1.1/go.mod h1:tAwRjCV0RjLTjH8DR/AU7VYvfQECg+lpUy2Mdzv7gyk= github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-routing v0.2.1/go.mod h1:xiNNiwgjmLqPS1cimvAw6EyB9rkVDbiocA4yY+wRNLM= -github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= github.com/ipfs/go-ipfs-routing v0.3.0/go.mod h1:dKqtTFIql7e1zYsEuWLyuOU+E0WJWW8JjbTPLParDWo= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= @@ -1259,8 +1257,6 @@ github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG github.com/ipfs/interface-go-ipfs-core v0.9.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0= github.com/ipfs/interface-go-ipfs-core v0.10.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0= github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE= -github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= -github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= github.com/ipld/go-car/v2 v2.5.1/go.mod h1:jKjGOqoCj5zn6KjnabD6JbnCsMntqU2hLiU6baZVO3E= github.com/ipld/go-car/v2 v2.8.0/go.mod h1:a+BnAxUqgr7wcWxW/lI6ctyEQ2v9gjBChPytwFMp2f4= @@ -1367,8 +1363,8 @@ github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/4 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= -github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= +github.com/klauspost/reedsolomon v1.12.1-0.20240110152930-bb8917fa442f h1:QEQvCKqgPSTRn9UIT65LSKY+7LCcGyiH6tIh6vCeHEw= +github.com/klauspost/reedsolomon v1.12.1-0.20240110152930-bb8917fa442f/go.mod h1:nEi5Kjb6QqtbofI6s+cbG/j1da11c96IBYBSnVGtuBs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= diff --git a/header/headertest/fraud/testing.go b/header/headertest/fraud/testing.go index e2ff13a4e0..6070cf0574 100644 --- a/header/headertest/fraud/testing.go +++ b/header/headertest/fraud/testing.go @@ -12,14 +12,13 @@ import ( "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/store" + "github.com/celestiaorg/celestia-node/share/testing/edstest" ) // FraudMaker allows to produce an invalid header at the specified height in order to produce the @@ -45,7 +44,7 @@ func NewFraudMaker(t *testing.T, height int64, vals []types.PrivValidator, valSe } } -func (f *FraudMaker) MakeExtendedHeader(odsSize int, edsStore *eds.Store) header.ConstructFn { +func (f *FraudMaker) MakeExtendedHeader(odsSize int, edsStore *store.Store) header.ConstructFn { return func( h *types.Header, comm *types.Commit, @@ -58,14 +57,14 @@ func (f *FraudMaker) MakeExtendedHeader(odsSize int, edsStore *eds.Store) header hdr := *h if h.Height == f.height { - adder := ipld.NewProofsAdder(odsSize) - square := edstest.RandByzantineEDS(f.t, odsSize, nmt.NodeVisitor(adder.VisitFn())) + square := edstest.RandByzantineEDS(f.t, odsSize) dah, err := da.NewDataAvailabilityHeader(square) require.NoError(f.t, err) hdr.DataHash = dah.Hash() - ctx := ipld.CtxWithProofsAdder(context.Background(), adder) - require.NoError(f.t, edsStore.Put(ctx, h.DataHash.Bytes(), square)) + file, err := edsStore.Put(context.Background(), dah.Hash(), uint64(h.Height), square) + require.NoError(f.t, err) + require.NoError(f.t, file.Close()) *eds = *square } diff --git a/header/headertest/testing.go b/header/headertest/testing.go index 7b0ae64262..1285e2cdd0 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -46,6 +46,14 @@ func NewStore(t *testing.T) libhead.Store[*header.ExtendedHeader] { return headertest.NewStore[*header.ExtendedHeader](t, NewTestSuite(t, 3, 0), 10) } +func NewCustomStore( + t *testing.T, + generator headertest.Generator[*header.ExtendedHeader], + numHeaders int, +) libhead.Store[*header.ExtendedHeader] { + return headertest.NewStore[*header.ExtendedHeader](t, generator, numHeaders) +} + // NewTestSuite setups a new test suite with a given number of validators. func NewTestSuite(t *testing.T, numValidators int, blockTime time.Duration) *TestSuite { valSet, vals := RandValidatorSet(numValidators, 10) @@ -82,8 +90,10 @@ func (s *TestSuite) genesis() *header.ExtendedHeader { return eh } -func MakeCommit(blockID types.BlockID, height int64, round int32, - voteSet *types.VoteSet, validators []types.PrivValidator, now time.Time) (*types.Commit, error) { +func MakeCommit( + blockID types.BlockID, height int64, round int32, + voteSet *types.VoteSet, validators []types.PrivValidator, now time.Time, +) (*types.Commit, error) { // all sign for i := 0; i < len(validators); i++ { @@ -157,7 +167,8 @@ func (s *TestSuite) NextHeader() *header.ExtendedHeader { } func (s *TestSuite) GenRawHeader( - height uint64, lastHeader, lastCommit, dataHash libhead.Hash) *header.RawHeader { + height uint64, lastHeader, lastCommit, dataHash libhead.Hash, +) *header.RawHeader { rh := RandRawHeader(s.t) rh.Height = int64(height) rh.LastBlockID = types.BlockID{Hash: bytes.HexBytes(lastHeader)} @@ -167,9 +178,9 @@ func (s *TestSuite) GenRawHeader( rh.NextValidatorsHash = s.valSet.Hash() rh.ProposerAddress = s.nextProposer().Address - rh.Time = time.Now() + rh.Time = time.Now().UTC() if s.blockTime > 0 { - rh.Time = s.Head().Time().Add(s.blockTime) + rh.Time = s.Head().Time().UTC().Add(s.blockTime) } return rh @@ -189,7 +200,7 @@ func (s *TestSuite) Commit(h *header.RawHeader) *types.Commit { ValidatorIndex: int32(i), Height: h.Height, Round: round, - Timestamp: tmtime.Now(), + Timestamp: tmtime.Now().UTC(), Type: tmproto.PrecommitType, BlockID: bid, } @@ -214,6 +225,11 @@ func (s *TestSuite) nextProposer() *types.Validator { // RandExtendedHeader provides an ExtendedHeader fixture. func RandExtendedHeader(t testing.TB) *header.ExtendedHeader { + timestamp := time.Now().UTC() + return RandExtendedHeaderAtTimestamp(t, timestamp) +} + +func RandExtendedHeaderAtTimestamp(t testing.TB, timestamp time.Time) *header.ExtendedHeader { dah := share.EmptyRoot() rh := RandRawHeader(t) @@ -224,7 +240,7 @@ func RandExtendedHeader(t testing.TB) *header.ExtendedHeader { voteSet := types.NewVoteSet(rh.ChainID, rh.Height, 0, tmproto.PrecommitType, valSet) blockID := RandBlockID(t) blockID.Hash = rh.Hash() - commit, err := MakeCommit(blockID, rh.Height, 0, voteSet, vals, time.Now()) + commit, err := MakeCommit(blockID, rh.Height, 0, voteSet, vals, timestamp) require.NoError(t, err) return &header.ExtendedHeader{ @@ -279,7 +295,7 @@ func RandRawHeader(t testing.TB) *header.RawHeader { Version: version.Consensus{Block: 11, App: 1}, ChainID: "test", Height: mrand.Int63(), //nolint:gosec - Time: time.Now(), + Time: time.Now().UTC(), LastBlockID: RandBlockID(t), LastCommitHash: tmrand.Bytes(32), DataHash: tmrand.Bytes(32), @@ -320,7 +336,7 @@ func ExtendedHeaderFromEDS(t testing.TB, height uint64, eds *rsmt2d.ExtendedData blockID := RandBlockID(t) blockID.Hash = gen.Hash() voteSet := types.NewVoteSet(gen.ChainID, gen.Height, 0, tmproto.PrecommitType, valSet) - commit, err := MakeCommit(blockID, gen.Height, 0, voteSet, vals, time.Now()) + commit, err := MakeCommit(blockID, gen.Height, 0, voteSet, vals, time.Now().UTC()) require.NoError(t, err) eh := &header.ExtendedHeader{ diff --git a/libs/edssser/edssser.go b/libs/edssser/edssser.go index 34712b785a..2012532b2e 100644 --- a/libs/edssser/edssser.go +++ b/libs/edssser/edssser.go @@ -14,7 +14,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/testing/edstest" ) type Config struct { diff --git a/libs/utils/logcloser.go b/libs/utils/logcloser.go new file mode 100644 index 0000000000..9027a50059 --- /dev/null +++ b/libs/utils/logcloser.go @@ -0,0 +1,13 @@ +package utils + +import ( + "io" + + "github.com/ipfs/go-log/v2" +) + +func CloseAndLog(log log.StandardLogger, name string, closer io.Closer) { + if err := closer.Close(); err != nil { + log.Warnf("closing %s: %s", name, err) + } +} diff --git a/nodebuilder/config.go b/nodebuilder/config.go index d323f401d7..bf9b1a5bfe 100644 --- a/nodebuilder/config.go +++ b/nodebuilder/config.go @@ -15,6 +15,7 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + "github.com/celestiaorg/celestia-node/nodebuilder/pruner" "github.com/celestiaorg/celestia-node/nodebuilder/rpc" "github.com/celestiaorg/celestia-node/nodebuilder/share" "github.com/celestiaorg/celestia-node/nodebuilder/state" @@ -35,6 +36,7 @@ type Config struct { Share share.Config Header header.Config DASer das.Config `toml:",omitempty"` + Pruner pruner.Config } // DefaultConfig provides a default Config for a given Node Type 'tp'. @@ -49,6 +51,7 @@ func DefaultConfig(tp node.Type) *Config { Gateway: gateway.DefaultConfig(), Share: share.DefaultConfig(tp), Header: header.DefaultConfig(tp), + Pruner: pruner.DefaultConfig(), } switch tp { diff --git a/nodebuilder/config_test.go b/nodebuilder/config_test.go index e7b64b0aed..3b98664025 100644 --- a/nodebuilder/config_test.go +++ b/nodebuilder/config_test.go @@ -97,7 +97,7 @@ var outdatedConfig = ` PeersLimit = 5 DiscoveryInterval = "30s" AdvertiseInterval = "30s" - UseShareExchange = true + UseShrEx = true [Share.ShrExEDSParams] ServerReadTimeout = "5s" ServerWriteTimeout = "1m0s" diff --git a/nodebuilder/core/module.go b/nodebuilder/core/module.go index fcf682cdf5..3e3de21d99 100644 --- a/nodebuilder/core/module.go +++ b/nodebuilder/core/module.go @@ -12,8 +12,8 @@ import ( "github.com/celestiaorg/celestia-node/libs/fxutil" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" + "github.com/celestiaorg/celestia-node/share/store" ) // ConstructModule collects all the components and services related to managing the relationship @@ -38,7 +38,7 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option fxutil.ProvideAs( func( fetcher *core.BlockFetcher, - store *eds.Store, + store *store.Store, construct header.ConstructFn, ) (*core.Exchange, error) { var opts []core.Option @@ -55,7 +55,7 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option fetcher *core.BlockFetcher, pubsub *shrexsub.PubSub, construct header.ConstructFn, - store *eds.Store, + store *store.Store, chainID p2p.Network, ) (*core.Listener, error) { opts := []core.Option{core.WithChainID(chainID)} diff --git a/nodebuilder/das/config.go b/nodebuilder/das/config.go index eeaa382a41..4d0276214f 100644 --- a/nodebuilder/das/config.go +++ b/nodebuilder/das/config.go @@ -23,6 +23,7 @@ func DefaultConfig(tp node.Type) Config { switch tp { case node.Light: cfg.SampleTimeout = modp2p.BlockTime * time.Duration(cfg.ConcurrencyLimit) + cfg.ConcurrencyLimit = 64 case node.Full: // Default value for DASer concurrency limit is based on dasing using ipld getter. // Full node will primarily use shrex protocol for sampling, that is much more efficient and can diff --git a/nodebuilder/module.go b/nodebuilder/module.go index ad287b1ac8..e3370eb083 100644 --- a/nodebuilder/module.go +++ b/nodebuilder/module.go @@ -16,7 +16,7 @@ import ( modhead "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" - "github.com/celestiaorg/celestia-node/nodebuilder/prune" + "github.com/celestiaorg/celestia-node/nodebuilder/pruner" "github.com/celestiaorg/celestia-node/nodebuilder/rpc" "github.com/celestiaorg/celestia-node/nodebuilder/share" "github.com/celestiaorg/celestia-node/nodebuilder/state" @@ -58,7 +58,7 @@ func ConstructModule(tp node.Type, network p2p.Network, cfg *Config, store Store blob.ConstructModule(), da.ConstructModule(), node.ConstructModule(tp), - prune.ConstructModule(tp), + pruner.ConstructModule(tp, &cfg.Pruner), rpc.ConstructModule(tp, &cfg.RPC), ) diff --git a/nodebuilder/node.go b/nodebuilder/node.go index b16a376cc1..ad19ba12d7 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -60,7 +60,7 @@ type Node struct { Host host.Host ConnGater *conngater.BasicConnectionGater Routing routing.PeerRouting - DataExchange exchange.Interface + DataExchange exchange.SessionExchange BlockService blockservice.BlockService // p2p protocols PubSub *pubsub.PubSub diff --git a/nodebuilder/p2p/bitswap.go b/nodebuilder/p2p/bitswap.go index 773d39ba4e..177627d5ae 100644 --- a/nodebuilder/p2p/bitswap.go +++ b/nodebuilder/p2p/bitswap.go @@ -16,7 +16,7 @@ import ( "github.com/libp2p/go-libp2p/core/protocol" "go.uber.org/fx" - "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/store" ) const ( @@ -29,7 +29,7 @@ const ( ) // dataExchange provides a constructor for IPFS block's DataExchange over BitSwap. -func dataExchange(params bitSwapParams) exchange.Interface { +func dataExchange(params bitSwapParams) exchange.SessionExchange { prefix := protocolID(params.Net) net := network.NewFromIpfsHost(params.Host, &routinghelpers.Null{}, network.Prefix(prefix)) srvr := server.New( @@ -76,10 +76,10 @@ func blockstoreFromDatastore(ctx context.Context, ds datastore.Batching) (blocks ) } -func blockstoreFromEDSStore(ctx context.Context, store *eds.Store) (blockstore.Blockstore, error) { +func blockstoreFromEDSStore(ctx context.Context, s *store.Store, ds datastore.Batching) (blockstore.Blockstore, error) { return blockstore.CachedBlockstore( ctx, - store.Blockstore(), + store.NewBlockstore(s, ds), blockstore.CacheOpts{ HasTwoQueueCacheSize: defaultARCCacheSize, }, @@ -97,5 +97,5 @@ type bitSwapParams struct { } func protocolID(network Network) protocol.ID { - return protocol.ID(fmt.Sprintf("/celestia/%s", network)) + return protocol.ID(fmt.Sprintf("/celestia/%s/v0.0.1", network)) } diff --git a/nodebuilder/prune/module.go b/nodebuilder/prune/module.go deleted file mode 100644 index 2141b74bf1..0000000000 --- a/nodebuilder/prune/module.go +++ /dev/null @@ -1,47 +0,0 @@ -package prune - -import ( - "context" - - "go.uber.org/fx" - - "github.com/celestiaorg/celestia-node/nodebuilder/node" - "github.com/celestiaorg/celestia-node/pruner" - "github.com/celestiaorg/celestia-node/pruner/archival" - "github.com/celestiaorg/celestia-node/pruner/light" -) - -func ConstructModule(tp node.Type) fx.Option { - baseComponents := fx.Options( - fx.Provide(fx.Annotate( - pruner.NewService, - fx.OnStart(func(ctx context.Context, p *pruner.Service) error { - return p.Start(ctx) - }), - fx.OnStop(func(ctx context.Context, p *pruner.Service) error { - return p.Stop(ctx) - }), - )), - ) - - switch tp { - case node.Full, node.Bridge: - return fx.Module("prune", - baseComponents, - fx.Provide(func() pruner.Pruner { - return archival.NewPruner() - }), - fx.Supply(archival.Window), - ) - case node.Light: - return fx.Module("prune", - baseComponents, - fx.Provide(func() pruner.Pruner { - return light.NewPruner() - }), - fx.Supply(light.Window), - ) - default: - panic("unknown node type") - } -} diff --git a/nodebuilder/pruner/config.go b/nodebuilder/pruner/config.go new file mode 100644 index 0000000000..1aa8c6ad6f --- /dev/null +++ b/nodebuilder/pruner/config.go @@ -0,0 +1,13 @@ +package pruner + +var MetricsEnabled bool + +type Config struct { + EnableService bool +} + +func DefaultConfig() Config { + return Config{ + EnableService: false, + } +} diff --git a/nodebuilder/pruner/constructors.go b/nodebuilder/pruner/constructors.go new file mode 100644 index 0000000000..1b84d19d0d --- /dev/null +++ b/nodebuilder/pruner/constructors.go @@ -0,0 +1,33 @@ +package pruner + +import ( + "github.com/ipfs/go-datastore" + + hdr "github.com/celestiaorg/go-header" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + "github.com/celestiaorg/celestia-node/pruner" +) + +func newPrunerService( + p pruner.Pruner, + window pruner.AvailabilityWindow, + getter hdr.Store[*header.ExtendedHeader], + ds datastore.Batching, + opts ...pruner.Option, +) (*pruner.Service, error) { + serv, err := pruner.NewService(p, window, getter, ds, p2p.BlockTime, opts...) + if err != nil { + return nil, err + } + + if MetricsEnabled { + err := pruner.WithPrunerMetrics(serv) + if err != nil { + return nil, err + } + } + + return serv, nil +} diff --git a/nodebuilder/pruner/flags.go b/nodebuilder/pruner/flags.go new file mode 100644 index 0000000000..7734c49e46 --- /dev/null +++ b/nodebuilder/pruner/flags.go @@ -0,0 +1,20 @@ +package pruner + +import ( + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" +) + +const pruningFlag = "experimental-pruning" + +func Flags() *flag.FlagSet { + flags := &flag.FlagSet{} + + flags.Bool(pruningFlag, false, "EXPERIMENTAL: Enables pruning of blocks outside the pruning window.") + + return flags +} + +func ParseFlags(cmd *cobra.Command, cfg *Config) { + cfg.EnableService = cmd.Flag(pruningFlag).Changed +} diff --git a/nodebuilder/pruner/module.go b/nodebuilder/pruner/module.go new file mode 100644 index 0000000000..ccb06d7188 --- /dev/null +++ b/nodebuilder/pruner/module.go @@ -0,0 +1,71 @@ +package pruner + +import ( + "context" + + "go.uber.org/fx" + + "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/celestia-node/pruner" + "github.com/celestiaorg/celestia-node/pruner/archival" + "github.com/celestiaorg/celestia-node/pruner/full" + "github.com/celestiaorg/celestia-node/pruner/light" + "github.com/celestiaorg/celestia-node/share/store" +) + +func ConstructModule(tp node.Type, cfg *Config) fx.Option { + if !cfg.EnableService { + switch tp { + case node.Light: + // light nodes are still subject to sampling within window + // even if pruning is not enabled. + return fx.Supply(light.Window) + case node.Full, node.Bridge: + return fx.Supply(archival.Window) + default: + panic("unknown node type") + } + } + + baseComponents := fx.Options( + fx.Provide(fx.Annotate( + newPrunerService, + fx.OnStart(func(ctx context.Context, p *pruner.Service) error { + return p.Start(ctx) + }), + fx.OnStop(func(ctx context.Context, p *pruner.Service) error { + return p.Stop(ctx) + }), + )), + // This is necessary to invoke the pruner service as independent thanks to a + // quirk in FX. + fx.Invoke(func(_ *pruner.Service) {}), + ) + + switch tp { + case node.Full: + return fx.Module("prune", + baseComponents, + fx.Provide(func(store *store.Store) pruner.Pruner { + return full.NewPruner(store) + }), + fx.Supply(full.Window), + ) + case node.Bridge: + return fx.Module("prune", + baseComponents, + fx.Provide(func(store *store.Store) pruner.Pruner { + return full.NewPruner(store) + }), + fx.Supply(full.Window), + ) + // TODO: Eventually, light nodes will be capable of pruning samples + // in which case, this can be enabled. + case node.Light: + return fx.Module("prune", + fx.Supply(light.Window), + ) + default: + panic("unknown node type") + } +} diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 7830f0e8f6..72a0b7c960 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -29,6 +29,7 @@ import ( modhead "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + modprune "github.com/celestiaorg/celestia-node/nodebuilder/pruner" "github.com/celestiaorg/celestia-node/nodebuilder/share" "github.com/celestiaorg/celestia-node/state" ) @@ -80,6 +81,7 @@ func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type) fx.Opti // control over which module to enable metrics for modhead.MetricsEnabled = true modcore.MetricsEnabled = true + modprune.MetricsEnabled = true baseComponents := fx.Options( fx.Supply(metricOpts), diff --git a/nodebuilder/share/config.go b/nodebuilder/share/config.go index 1d984b6dca..970cac8238 100644 --- a/nodebuilder/share/config.go +++ b/nodebuilder/share/config.go @@ -5,19 +5,20 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/share/availability/light" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" + "github.com/celestiaorg/celestia-node/share/store" ) // TODO: some params are pointers and other are not, Let's fix this. type Config struct { // EDSStoreParams sets eds store configuration parameters - EDSStoreParams *eds.Parameters + EDSStoreParams *store.Parameters - UseShareExchange bool + UseShrEx bool + UseShwap bool // ShrExEDSParams sets shrexeds client and server configuration parameters ShrExEDSParams *shrexeds.Parameters // ShrExNDParams sets shrexnd client and server configuration parameters @@ -31,11 +32,11 @@ type Config struct { func DefaultConfig(tp node.Type) Config { cfg := Config{ - EDSStoreParams: eds.DefaultParameters(), + EDSStoreParams: store.DefaultParameters(), Discovery: discovery.DefaultParameters(), ShrExEDSParams: shrexeds.DefaultParameters(), ShrExNDParams: shrexnd.DefaultParameters(), - UseShareExchange: true, + UseShrEx: true, PeerManagerParams: peers.DefaultParameters(), } diff --git a/nodebuilder/share/constructors.go b/nodebuilder/share/constructors.go index 96be2b5d20..ed4578f712 100644 --- a/nodebuilder/share/constructors.go +++ b/nodebuilder/share/constructors.go @@ -1,28 +1,20 @@ package share import ( - "context" - "errors" - - "github.com/filecoin-project/dagstore" - "github.com/ipfs/boxo/blockservice" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/routing" routingdisc "github.com/libp2p/go-libp2p/p2p/discovery/routing" - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/getters" - "github.com/celestiaorg/celestia-node/share/ipld" disc "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/peers" + shwap_getter "github.com/celestiaorg/celestia-node/share/shwap/getter" ) const ( // fullNodesTag is the tag used to identify full nodes in the discovery service. - fullNodesTag = "full" + fullNodesTag = "full/v0.0.1" ) func newDiscovery(cfg *disc.Parameters, @@ -46,40 +38,20 @@ func newShareModule(getter share.Getter, avail share.Availability) Module { return &module{getter, avail} } -// ensureEmptyCARExists adds an empty EDS to the provided EDS store. -func ensureEmptyCARExists(ctx context.Context, store *eds.Store) error { - emptyEDS := share.EmptyExtendedDataSquare() - emptyDAH, err := da.NewDataAvailabilityHeader(emptyEDS) - if err != nil { - return err - } - - err = store.Put(ctx, emptyDAH.Hash(), emptyEDS) - if errors.Is(err, dagstore.ErrShardExists) { - return nil - } - return err -} - -// ensureEmptyEDSInBS checks if the given DAG contains an empty block data square. -// If it does not, it stores an empty block. This optimization exists to prevent -// redundant storing of empty block data so that it is only stored once and returned -// upon request for a block with an empty data square. -func ensureEmptyEDSInBS(ctx context.Context, bServ blockservice.BlockService) error { - _, err := ipld.AddShares(ctx, share.EmptyBlockShares(), bServ) - return err -} - func lightGetter( shrexGetter *getters.ShrexGetter, - ipldGetter *getters.IPLDGetter, + shwapGetter *shwap_getter.Getter, + reconstructGetter *shwap_getter.ReconstructionGetter, cfg Config, ) share.Getter { var cascade []share.Getter - if cfg.UseShareExchange { + if cfg.UseShrEx { cascade = append(cascade, shrexGetter) } - cascade = append(cascade, ipldGetter) + if cfg.UseShwap { + cascade = append(cascade, shwapGetter) + } + cascade = append(cascade, reconstructGetter) return getters.NewCascadeGetter(cascade) } @@ -94,7 +66,7 @@ func bridgeGetter( ) share.Getter { var cascade []share.Getter cascade = append(cascade, storeGetter) - if cfg.UseShareExchange { + if cfg.UseShrEx { cascade = append(cascade, shrexGetter) } return getters.NewCascadeGetter(cascade) @@ -103,14 +75,18 @@ func bridgeGetter( func fullGetter( storeGetter *getters.StoreGetter, shrexGetter *getters.ShrexGetter, - ipldGetter *getters.IPLDGetter, + shwapGetter *shwap_getter.Getter, + reconstructGetter *shwap_getter.ReconstructionGetter, cfg Config, ) share.Getter { var cascade []share.Getter cascade = append(cascade, storeGetter) - if cfg.UseShareExchange { + if cfg.UseShrEx { cascade = append(cascade, shrexGetter) } - cascade = append(cascade, ipldGetter) + if cfg.UseShwap { + cascade = append(cascade, shwapGetter) + } + cascade = append(cascade, reconstructGetter) return getters.NewCascadeGetter(cascade) } diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index b81149535f..3792012fd8 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -3,7 +3,6 @@ package share import ( "context" - "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/p2p/net/conngater" "go.uber.org/fx" @@ -17,13 +16,14 @@ import ( "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/availability/full" "github.com/celestiaorg/celestia-node/share/availability/light" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/getters" disc "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" + shwap_getter "github.com/celestiaorg/celestia-node/share/shwap/getter" + "github.com/celestiaorg/celestia-node/share/store" ) func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option { @@ -74,7 +74,8 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option "share", baseComponents, bridgeAndFullComponents, - fx.Provide(getters.NewIPLDGetter), + fx.Provide(shwap_getter.NewGetter), + fx.Provide(shwap_getter.NewReconstructionGetter), fx.Provide(fullGetter), ) case node.Light: @@ -83,8 +84,8 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option baseComponents, shrexGetterComponents(cfg), lightAvailabilityComponents(cfg), - fx.Invoke(ensureEmptyEDSInBS), - fx.Provide(getters.NewIPLDGetter), + fx.Provide(shwap_getter.NewGetter), + fx.Provide(shwap_getter.NewReconstructionGetter), fx.Provide(lightGetter), // shrexsub broadcaster stub for daser fx.Provide(func() shrexsub.BroadcastFn { @@ -191,7 +192,7 @@ func shrexServerComponents(cfg *Config) fx.Option { return fx.Options( fx.Invoke(func(_ *shrexeds.Server, _ *shrexnd.Server) {}), fx.Provide(fx.Annotate( - func(host host.Host, store *eds.Store, network modp2p.Network) (*shrexeds.Server, error) { + func(host host.Host, store *store.Store, network modp2p.Network) (*shrexeds.Server, error) { cfg.ShrExEDSParams.WithNetworkID(network.String()) return shrexeds.NewServer(cfg.ShrExEDSParams, host, store) }, @@ -205,7 +206,7 @@ func shrexServerComponents(cfg *Config) fx.Option { fx.Provide(fx.Annotate( func( host host.Host, - store *eds.Store, + store *store.Store, network modp2p.Network, ) (*shrexnd.Server, error) { cfg.ShrExNDParams.WithNetworkID(network.String()) @@ -224,19 +225,9 @@ func shrexServerComponents(cfg *Config) fx.Option { func edsStoreComponents(cfg *Config) fx.Option { return fx.Options( fx.Provide(fx.Annotate( - func(path node.StorePath, ds datastore.Batching) (*eds.Store, error) { - return eds.NewStore(cfg.EDSStoreParams, string(path), ds) + func(path node.StorePath) (*store.Store, error) { + return store.NewStore(cfg.EDSStoreParams, string(path)) }, - fx.OnStart(func(ctx context.Context, store *eds.Store) error { - err := store.Start(ctx) - if err != nil { - return err - } - return ensureEmptyCARExists(ctx, store) - }), - fx.OnStop(func(ctx context.Context, store *eds.Store) error { - return store.Stop(ctx) - }), )), ) } diff --git a/nodebuilder/share/opts.go b/nodebuilder/share/opts.go index e236847f41..d0d381d5f4 100644 --- a/nodebuilder/share/opts.go +++ b/nodebuilder/share/opts.go @@ -1,12 +1,12 @@ package share import ( - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/getters" disc "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" + "github.com/celestiaorg/celestia-node/share/store" ) // WithPeerManagerMetrics is a utility function to turn on peer manager metrics and that is @@ -43,6 +43,6 @@ func WithShrexGetterMetrics(sg *getters.ShrexGetter) error { return sg.WithMetrics() } -func WithStoreMetrics(s *eds.Store) error { +func WithStoreMetrics(s *store.Store) error { return s.WithMetrics() } diff --git a/nodebuilder/share/share_test.go b/nodebuilder/share/share_test.go deleted file mode 100644 index db170709db..0000000000 --- a/nodebuilder/share/share_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package share - -import ( - "context" - "testing" - - "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" -) - -func Test_EmptyCARExists(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := eds.NewStore(eds.DefaultParameters(), t.TempDir(), ds) - require.NoError(t, err) - err = edsStore.Start(ctx) - require.NoError(t, err) - - eds := share.EmptyExtendedDataSquare() - dah, err := share.NewRoot(eds) - require.NoError(t, err) - - // add empty EDS to store - err = ensureEmptyCARExists(ctx, edsStore) - assert.NoError(t, err) - - // assert that the empty car exists - has, err := edsStore.Has(ctx, dah.Hash()) - assert.True(t, has) - assert.NoError(t, err) - - // assert that the empty car is, in fact, empty - emptyEds, err := edsStore.Get(ctx, dah.Hash()) - assert.Equal(t, eds.Flattened(), emptyEds.Flattened()) - assert.NoError(t, err) -} diff --git a/nodebuilder/store_test.go b/nodebuilder/store_test.go index 51bd89c5a7..7a208ae479 100644 --- a/nodebuilder/store_test.go +++ b/nodebuilder/store_test.go @@ -3,25 +3,13 @@ package nodebuilder import ( - "context" "strconv" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/wrapper" - "github.com/celestiaorg/nmt" - "github.com/celestiaorg/rsmt2d" - "github.com/celestiaorg/celestia-node/nodebuilder/node" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/ipld" - "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestRepo(t *testing.T) { @@ -64,123 +52,3 @@ func TestRepo(t *testing.T) { }) } } - -func BenchmarkStore(b *testing.B) { - ctx, cancel := context.WithCancel(context.Background()) - b.Cleanup(cancel) - - // BenchmarkStore/bench_read_128-10 14 78970661 ns/op (~70ms) - b.Run("bench put 128", func(b *testing.B) { - dir := b.TempDir() - err := Init(*DefaultConfig(node.Full), dir, node.Full) - require.NoError(b, err) - - store := newStore(ctx, b, eds.DefaultParameters(), dir) - size := 128 - b.Run("enabled eds proof caching", func(b *testing.B) { - b.StopTimer() - b.ResetTimer() - for i := 0; i < b.N; i++ { - adder := ipld.NewProofsAdder(size * 2) - shares := sharetest.RandShares(b, size*size) - eds, err := rsmt2d.ComputeExtendedDataSquare( - shares, - share.DefaultRSMT2DCodec(), - wrapper.NewConstructor(uint64(size), - nmt.NodeVisitor(adder.VisitFn())), - ) - require.NoError(b, err) - dah, err := da.NewDataAvailabilityHeader(eds) - require.NoError(b, err) - ctx := ipld.CtxWithProofsAdder(ctx, adder) - - b.StartTimer() - err = store.edsStore.Put(ctx, dah.Hash(), eds) - b.StopTimer() - require.NoError(b, err) - } - }) - - b.Run("disabled eds proof caching", func(b *testing.B) { - b.ResetTimer() - b.StopTimer() - for i := 0; i < b.N; i++ { - eds := edstest.RandEDS(b, size) - dah, err := da.NewDataAvailabilityHeader(eds) - require.NoError(b, err) - - b.StartTimer() - err = store.edsStore.Put(ctx, dah.Hash(), eds) - b.StopTimer() - require.NoError(b, err) - } - }) - }) -} - -func TestStoreRestart(t *testing.T) { - const ( - blocks = 5 - size = 32 - ) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - t.Cleanup(cancel) - - dir := t.TempDir() - err := Init(*DefaultConfig(node.Full), dir, node.Full) - require.NoError(t, err) - - store := newStore(ctx, t, eds.DefaultParameters(), dir) - - hashes := make([][]byte, blocks) - for i := range hashes { - edss := edstest.RandEDS(t, size) - require.NoError(t, err) - dah, err := da.NewDataAvailabilityHeader(edss) - require.NoError(t, err) - err = store.edsStore.Put(ctx, dah.Hash(), edss) - require.NoError(t, err) - - // store hashes for read loop later - hashes[i] = dah.Hash() - } - - // restart store - store.stop(ctx, t) - store = newStore(ctx, t, eds.DefaultParameters(), dir) - - for _, h := range hashes { - edsReader, err := store.edsStore.GetCAR(ctx, h) - require.NoError(t, err) - odsReader, err := eds.ODSReader(edsReader) - require.NoError(t, err) - _, err = eds.ReadEDS(ctx, odsReader, h) - require.NoError(t, err) - require.NoError(t, edsReader.Close()) - } -} - -type store struct { - s Store - edsStore *eds.Store -} - -func newStore(ctx context.Context, t require.TestingT, params *eds.Parameters, dir string) store { - s, err := OpenStore(dir, nil) - require.NoError(t, err) - ds, err := s.Datastore() - require.NoError(t, err) - edsStore, err := eds.NewStore(params, dir, ds) - require.NoError(t, err) - err = edsStore.Start(ctx) - require.NoError(t, err) - return store{ - s: s, - edsStore: edsStore, - } -} - -func (s *store) stop(ctx context.Context, t *testing.T) { - require.NoError(t, s.edsStore.Stop(ctx)) - require.NoError(t, s.s.Close()) -} diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index 6496cdbb53..1296fab39f 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -7,8 +7,6 @@ import ( "testing" "time" - "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" @@ -23,8 +21,8 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/core" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/byzantine" + "github.com/celestiaorg/celestia-node/share/store" ) /* @@ -60,14 +58,9 @@ func TestFraudProofHandling(t *testing.T) { set, val := sw.Validators(t) fMaker := headerfraud.NewFraudMaker(t, 10, []types.PrivValidator{val}, set) - storeCfg := eds.DefaultParameters() - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := eds.NewStore(storeCfg, t.TempDir(), ds) + storeCfg := store.DefaultParameters() + edsStore, err := store.NewStore(storeCfg, t.TempDir()) require.NoError(t, err) - require.NoError(t, edsStore.Start(ctx)) - t.Cleanup(func() { - _ = edsStore.Stop(ctx) - }) cfg := nodebuilder.DefaultConfig(node.Bridge) // 1. @@ -86,7 +79,7 @@ func TestFraudProofHandling(t *testing.T) { addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) require.NoError(t, err) cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) - cfg.Share.UseShareExchange = false + cfg.Share.UseShrEx = false store := nodebuilder.MockStore(t, cfg) full := sw.NewNodeWithStore(node.Full, store) diff --git a/nodebuilder/tests/nd_test.go b/nodebuilder/tests/nd_test.go index 338aa6d0c1..ea807fa8b2 100644 --- a/nodebuilder/tests/nd_test.go +++ b/nodebuilder/tests/nd_test.go @@ -17,9 +17,9 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/p2p" "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/getters" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" + "github.com/celestiaorg/celestia-node/share/store" ) func TestShrexNDFromLights(t *testing.T) { @@ -177,7 +177,7 @@ func replaceNDServer(cfg *nodebuilder.Config, handler network.StreamHandler) fx. return fx.Decorate(fx.Annotate( func( host host.Host, - store *eds.Store, + store *store.Store, network p2p.Network, ) (*shrexnd.Server, error) { cfg.Share.ShrExNDParams.WithNetworkID(network.String()) @@ -198,7 +198,7 @@ func replaceShareGetter() fx.Option { return fx.Decorate(fx.Annotate( func( host host.Host, - store *eds.Store, + store *store.Store, storeGetter *getters.StoreGetter, shrexGetter *getters.ShrexGetter, network p2p.Network, diff --git a/nodebuilder/tests/reconstruct_test.go b/nodebuilder/tests/reconstruct_test.go index d047182669..6b45d90dfa 100644 --- a/nodebuilder/tests/reconstruct_test.go +++ b/nodebuilder/tests/reconstruct_test.go @@ -58,7 +58,7 @@ func TestFullReconstructFromBridge(t *testing.T) { require.NoError(t, err) cfg := nodebuilder.DefaultConfig(node.Full) - cfg.Share.UseShareExchange = false + cfg.Share.UseShrEx = false cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, getMultiAddr(t, bridge.Host)) full := sw.NewNodeWithConfig(node.Full, cfg) err = full.Start(ctx) @@ -170,7 +170,7 @@ func TestFullReconstructFromFulls(t *testing.T) { cfg := nodebuilder.DefaultConfig(node.Full) setTimeInterval(cfg, defaultTimeInterval) - cfg.Share.UseShareExchange = false + cfg.Share.UseShrEx = false cfg.Share.Discovery.PeersLimit = 0 cfg.Header.TrustedPeers = []string{lnBootstrapper1[0].String()} full1 := sw.NewNodeWithConfig(node.Full, cfg) @@ -301,7 +301,7 @@ func TestFullReconstructFromLights(t *testing.T) { cfg = nodebuilder.DefaultConfig(node.Full) setTimeInterval(cfg, defaultTimeInterval) - cfg.Share.UseShareExchange = false + cfg.Share.UseShrEx = false cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrsBridge[0].String()) nodesConfig := nodebuilder.WithBootstrappers([]peer.AddrInfo{*bootstrapperAddr}) full := sw.NewNodeWithConfig(node.Full, cfg, nodesConfig) diff --git a/nodebuilder/tests/swamp/swamp.go b/nodebuilder/tests/swamp/swamp.go index 9faf69744d..b71ea52df0 100644 --- a/nodebuilder/tests/swamp/swamp.go +++ b/nodebuilder/tests/swamp/swamp.go @@ -9,8 +9,6 @@ import ( "testing" "time" - ds "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" @@ -34,7 +32,7 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" "github.com/celestiaorg/celestia-node/nodebuilder/state" - "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/store" ) var blackholeIP6 = net.ParseIP("100::") @@ -173,8 +171,7 @@ func (s *Swamp) setupGenesis() { // ensure core has surpassed genesis block s.WaitTillHeight(ctx, 2) - ds := ds_sync.MutexWrap(ds.NewMapDatastore()) - store, err := eds.NewStore(eds.DefaultParameters(), s.t.TempDir(), ds) + store, err := store.NewStore(store.DefaultParameters(), s.t.TempDir()) require.NoError(s.t, err) ex, err := core.NewExchange( @@ -288,7 +285,7 @@ func (s *Swamp) newNode(t node.Type, store nodebuilder.Store, options ...fx.Opti cfg, _ := store.Config() cfg.RPC.Port = "0" - // tempDir is used for the eds.Store + // tempDir is used for the store.Store tempDir := s.t.TempDir() options = append(options, p2p.WithHost(s.createPeer(ks)), diff --git a/pruner/archival/pruner.go b/pruner/archival/pruner.go index 7b1cb935f3..a1a55db0da 100644 --- a/pruner/archival/pruner.go +++ b/pruner/archival/pruner.go @@ -15,6 +15,6 @@ func NewPruner() *Pruner { return &Pruner{} } -func (p *Pruner) Prune(context.Context, ...*header.ExtendedHeader) error { +func (p *Pruner) Prune(context.Context, *header.ExtendedHeader) error { return nil } diff --git a/pruner/checkpoint.go b/pruner/checkpoint.go new file mode 100644 index 0000000000..10db918cb5 --- /dev/null +++ b/pruner/checkpoint.go @@ -0,0 +1,73 @@ +package pruner + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/ipfs/go-datastore" + + "github.com/celestiaorg/celestia-node/header" +) + +var ( + storePrefix = datastore.NewKey("pruner") + checkpointKey = datastore.NewKey("checkpoint") +) + +// checkpoint contains information related to the state of the +// pruner service that is periodically persisted to disk. +type checkpoint struct { + LastPrunedHeight uint64 `json:"last_pruned_height"` + FailedHeaders map[uint64]struct{} `json:"failed"` +} + +// initializeCheckpoint initializes the checkpoint, storing the earliest header in the chain. +func (s *Service) initializeCheckpoint(ctx context.Context) error { + return s.updateCheckpoint(ctx, uint64(1), nil) +} + +// loadCheckpoint loads the last checkpoint from disk, initializing it if it does not already exist. +func (s *Service) loadCheckpoint(ctx context.Context) error { + bin, err := s.ds.Get(ctx, checkpointKey) + if err != nil { + if err == datastore.ErrNotFound { + return s.initializeCheckpoint(ctx) + } + return fmt.Errorf("failed to load checkpoint: %w", err) + } + + var cp *checkpoint + err = json.Unmarshal(bin, &cp) + if err != nil { + return fmt.Errorf("failed to unmarshal checkpoint: %w", err) + } + + s.checkpoint = cp + return nil +} + +// updateCheckpoint updates the checkpoint with the last pruned header height +// and persists it to disk. +func (s *Service) updateCheckpoint( + ctx context.Context, + lastPrunedHeight uint64, + failedHeights map[uint64]struct{}, +) error { + for height := range failedHeights { + s.checkpoint.FailedHeaders[height] = struct{}{} + } + + s.checkpoint.LastPrunedHeight = lastPrunedHeight + + bin, err := json.Marshal(s.checkpoint) + if err != nil { + return err + } + + return s.ds.Put(ctx, checkpointKey, bin) +} + +func (s *Service) lastPruned(ctx context.Context) (*header.ExtendedHeader, error) { + return s.getter.GetByHeight(ctx, s.checkpoint.LastPrunedHeight) +} diff --git a/pruner/find.go b/pruner/find.go new file mode 100644 index 0000000000..5091c168a0 --- /dev/null +++ b/pruner/find.go @@ -0,0 +1,114 @@ +package pruner + +import ( + "context" + "time" + + "github.com/celestiaorg/celestia-node/header" +) + +// maxHeadersPerLoop is the maximum number of headers to fetch +// for a prune loop (prevents fetching too many headers at a +// time for nodes that have a large number of pruneable headers). +var maxHeadersPerLoop = uint64(512) + +// findPruneableHeaders returns all headers that are eligible for pruning +// (outside the sampling window). +func (s *Service) findPruneableHeaders( + ctx context.Context, + lastPruned *header.ExtendedHeader, +) ([]*header.ExtendedHeader, error) { + pruneCutoff := time.Now().UTC().Add(time.Duration(-s.window)) + + if !lastPruned.Time().UTC().Before(pruneCutoff) { + // this can happen when the network is young and all blocks + // are still within the AvailabilityWindow + return nil, nil + } + + estimatedCutoffHeight, err := s.calculateEstimatedCutoff(ctx, lastPruned, pruneCutoff) + if err != nil { + return nil, err + } + + if lastPruned.Height() == estimatedCutoffHeight { + // nothing left to prune + return nil, nil + } + + log.Debugw("finder: fetching header range", "last pruned", lastPruned.Height(), + "target height", estimatedCutoffHeight) + + headers, err := s.getter.GetRangeByHeight(ctx, lastPruned, estimatedCutoffHeight) + if err != nil { + log.Errorw("failed to get range from header store", "from", lastPruned.Height(), + "to", estimatedCutoffHeight, "error", err) + return nil, err + } + // ensures genesis block gets pruned + if lastPruned.Height() == 1 { + headers = append([]*header.ExtendedHeader{lastPruned}, headers...) + } + + // if our estimated range didn't cover enough headers, we need to fetch more + // TODO: This is really inefficient in the case that lastPruned is the default value, or if the + // node has been offline for a long time. Instead of increasing the boundary by one in the for + // loop we could increase by a range every iteration + headerCount := len(headers) + for { + if headerCount > int(maxHeadersPerLoop) { + headers = headers[:maxHeadersPerLoop] + break + } + lastHeader := headers[len(headers)-1] + if lastHeader.Time().After(pruneCutoff) { + break + } + + nextHeader, err := s.getter.GetByHeight(ctx, lastHeader.Height()+1) + if err != nil { + log.Errorw("failed to get header by height", "height", lastHeader.Height()+1, "error", err) + return nil, err + } + headers = append(headers, nextHeader) + headerCount++ + } + + for i, h := range headers { + if h.Time().After(pruneCutoff) { + if i == 0 { + // we can't prune anything + return nil, nil + } + + // we can ignore the rest of the headers since they are all newer than the cutoff + return headers[:i], nil + } + } + return headers, nil +} + +func (s *Service) calculateEstimatedCutoff( + ctx context.Context, + lastPruned *header.ExtendedHeader, + pruneCutoff time.Time, +) (uint64, error) { + estimatedRange := uint64(pruneCutoff.UTC().Sub(lastPruned.Time().UTC()) / s.blockTime) + estimatedCutoffHeight := lastPruned.Height() + estimatedRange + + head, err := s.getter.Head(ctx) + if err != nil { + log.Errorw("failed to get Head from header store", "error", err) + return 0, err + } + + if head.Height() < estimatedCutoffHeight { + estimatedCutoffHeight = head.Height() + } + + if estimatedCutoffHeight-lastPruned.Height() > maxHeadersPerLoop { + estimatedCutoffHeight = lastPruned.Height() + maxHeadersPerLoop + } + + return estimatedCutoffHeight, nil +} diff --git a/pruner/full/pruner.go b/pruner/full/pruner.go new file mode 100644 index 0000000000..f2f687e45a --- /dev/null +++ b/pruner/full/pruner.go @@ -0,0 +1,40 @@ +package full + +import ( + "context" + "errors" + + "github.com/filecoin-project/dagstore" + logging "github.com/ipfs/go-log/v2" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/store" +) + +var log = logging.Logger("pruner/full") + +type Pruner struct { + store *store.Store +} + +func NewPruner(store *store.Store) *Pruner { + return &Pruner{ + store: store, + } +} + +func (p *Pruner) Prune(ctx context.Context, eh *header.ExtendedHeader) error { + // short circuit on empty roots + if eh.DAH.Equals(share.EmptyRoot()) { + return nil + } + + log.Debugf("pruning header %s", eh.DAH.Hash()) + + err := p.store.Remove(ctx, eh.Height()) + if err != nil && !errors.Is(err, dagstore.ErrShardUnknown) { + return err + } + return nil +} diff --git a/pruner/full/window.go b/pruner/full/window.go new file mode 100644 index 0000000000..4ad69234e2 --- /dev/null +++ b/pruner/full/window.go @@ -0,0 +1,12 @@ +package full + +import ( + "time" + + "github.com/celestiaorg/celestia-node/pruner" + "github.com/celestiaorg/celestia-node/pruner/light" +) + +// Window is the availability window for light nodes in the Celestia +// network (30 days + 1 hour). +const Window = pruner.AvailabilityWindow(time.Duration(light.Window) + time.Hour) diff --git a/pruner/light/pruner.go b/pruner/light/pruner.go index 513bfa2b66..61401bae74 100644 --- a/pruner/light/pruner.go +++ b/pruner/light/pruner.go @@ -12,6 +12,6 @@ func NewPruner() *Pruner { return &Pruner{} } -func (p *Pruner) Prune(context.Context, ...*header.ExtendedHeader) error { +func (p *Pruner) Prune(context.Context, *header.ExtendedHeader) error { return nil } diff --git a/pruner/light/window.go b/pruner/light/window.go index dc1a9e4444..2241ecb063 100644 --- a/pruner/light/window.go +++ b/pruner/light/window.go @@ -1,11 +1,9 @@ package light import ( - "time" - "github.com/celestiaorg/celestia-node/pruner" ) // Window is the availability window for light nodes in the Celestia // network (30 days). -const Window = pruner.AvailabilityWindow(time.Second * 86400 * 30) +const Window = pruner.AvailabilityWindow(30 * 24 * 60 * 60) diff --git a/pruner/metrics.go b/pruner/metrics.go new file mode 100644 index 0000000000..c43217dc3d --- /dev/null +++ b/pruner/metrics.go @@ -0,0 +1,80 @@ +package pruner + +import ( + "context" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var ( + meter = otel.Meter("storage_pruner") +) + +type metrics struct { + prunedCounter metric.Int64Counter + + lastPruned metric.Int64ObservableGauge + failedPrunes metric.Int64ObservableGauge + + clientReg metric.Registration +} + +func (s *Service) WithMetrics() error { + prunedCounter, err := meter.Int64Counter("prnr_pruned_counter", + metric.WithDescription("pruner pruned header counter")) + if err != nil { + return err + } + + failedPrunes, err := meter.Int64ObservableGauge("prnr_failed_counter", + metric.WithDescription("pruner failed prunes counter")) + if err != nil { + return err + } + + lastPruned, err := meter.Int64ObservableGauge("prnr_last_pruned", + metric.WithDescription("pruner highest pruned height")) + if err != nil { + return err + } + + callback := func(_ context.Context, observer metric.Observer) error { + observer.ObserveInt64(lastPruned, int64(s.checkpoint.LastPrunedHeight)) + observer.ObserveInt64(failedPrunes, int64(len(s.checkpoint.FailedHeaders))) + return nil + } + + clientReg, err := meter.RegisterCallback(callback, lastPruned, failedPrunes) + if err != nil { + return err + } + + s.metrics = &metrics{ + prunedCounter: prunedCounter, + lastPruned: lastPruned, + failedPrunes: failedPrunes, + clientReg: clientReg, + } + return nil +} + +func (m *metrics) close() error { + if m == nil { + return nil + } + + return m.clientReg.Unregister() +} + +func (m *metrics) observePrune(ctx context.Context, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + m.prunedCounter.Add(ctx, 1, metric.WithAttributes( + attribute.Bool("failed", failed))) +} diff --git a/pruner/params.go b/pruner/params.go new file mode 100644 index 0000000000..253ea5e1a9 --- /dev/null +++ b/pruner/params.go @@ -0,0 +1,41 @@ +package pruner + +import ( + "fmt" + "time" +) + +type Option func(*Params) + +type Params struct { + // pruneCycle is the frequency at which the pruning Service + // runs the ticker. If set to 0, the Service will not run. + pruneCycle time.Duration +} + +func (p *Params) Validate() error { + if p.pruneCycle == time.Duration(0) { + return fmt.Errorf("invalid GC cycle given, value should be positive and non-zero") + } + return nil +} + +func DefaultParams() Params { + return Params{ + pruneCycle: time.Minute * 5, + } +} + +// WithPruneCycle configures how often the pruning Service +// triggers a pruning cycle. +func WithPruneCycle(cycle time.Duration) Option { + return func(p *Params) { + p.pruneCycle = cycle + } +} + +// WithPrunerMetrics is a utility function to turn on pruner metrics and that is +// expected to be "invoked" by the fx lifecycle. +func WithPrunerMetrics(s *Service) error { + return s.WithMetrics() +} diff --git a/pruner/pruner.go b/pruner/pruner.go index fae60e483c..a591a65392 100644 --- a/pruner/pruner.go +++ b/pruner/pruner.go @@ -9,5 +9,5 @@ import ( // Pruner contains methods necessary to prune data // from the node's datastore. type Pruner interface { - Prune(context.Context, ...*header.ExtendedHeader) error + Prune(context.Context, *header.ExtendedHeader) error } diff --git a/pruner/service.go b/pruner/service.go index f67265977a..65935e75d8 100644 --- a/pruner/service.go +++ b/pruner/service.go @@ -2,24 +2,191 @@ package pruner import ( "context" + "fmt" + "time" + + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + logging "github.com/ipfs/go-log/v2" + + hdr "github.com/celestiaorg/go-header" + + "github.com/celestiaorg/celestia-node/header" ) -// Service handles the pruning routine for the node using the -// prune Pruner. +var log = logging.Logger("pruner/service") + +// Service handles running the pruning cycle for the node. type Service struct { pruner Pruner + window AvailabilityWindow + + getter hdr.Getter[*header.ExtendedHeader] + + ds datastore.Datastore + checkpoint *checkpoint + + blockTime time.Duration + + ctx context.Context + cancel context.CancelFunc + doneCh chan struct{} + + params Params + metrics *metrics } -func NewService(p Pruner) *Service { - return &Service{ - pruner: p, +func NewService( + p Pruner, + window AvailabilityWindow, + getter hdr.Getter[*header.ExtendedHeader], + ds datastore.Datastore, + blockTime time.Duration, + opts ...Option, +) (*Service, error) { + params := DefaultParams() + for _, opt := range opts { + opt(¶ms) + } + + if err := params.Validate(); err != nil { + return nil, err } + + return &Service{ + pruner: p, + window: window, + getter: getter, + checkpoint: &checkpoint{FailedHeaders: map[uint64]struct{}{}}, + ds: namespace.Wrap(ds, storePrefix), + blockTime: blockTime, + doneCh: make(chan struct{}), + params: params, + }, nil } +// Start loads the pruner's last pruned height (1 if pruner is freshly +// initialized) and runs the prune loop, pruning any blocks older than +// the given availability window. func (s *Service) Start(context.Context) error { + s.ctx, s.cancel = context.WithCancel(context.Background()) + + err := s.loadCheckpoint(s.ctx) + if err != nil { + return err + } + log.Debugw("loaded checkpoint", "lastPruned", s.checkpoint.LastPrunedHeight) + + go s.run() return nil } -func (s *Service) Stop(context.Context) error { - return nil +func (s *Service) Stop(ctx context.Context) error { + s.cancel() + + s.metrics.close() + + select { + case <-s.doneCh: + return nil + case <-ctx.Done(): + return fmt.Errorf("pruner unable to exit within context deadline") + } +} + +// run prunes blocks older than the availability wiindow periodically until the +// pruner service is stopped. +func (s *Service) run() { + defer close(s.doneCh) + + ticker := time.NewTicker(s.params.pruneCycle) + defer ticker.Stop() + + lastPrunedHeader, err := s.lastPruned(s.ctx) + if err != nil { + log.Errorw("failed to get last pruned header", "height", s.checkpoint.LastPrunedHeight, + "err", err) + log.Warn("exiting pruner service!") + + s.cancel() + } + + for { + select { + case <-s.ctx.Done(): + return + case <-ticker.C: + lastPrunedHeader = s.prune(s.ctx, lastPrunedHeader) + } + } +} + +func (s *Service) prune( + ctx context.Context, + lastPrunedHeader *header.ExtendedHeader, +) *header.ExtendedHeader { + // prioritize retrying previously-failed headers + s.retryFailed(s.ctx) + + for { + select { + case <-s.ctx.Done(): + return lastPrunedHeader + default: + } + + headers, err := s.findPruneableHeaders(ctx, lastPrunedHeader) + if err != nil || len(headers) == 0 { + return lastPrunedHeader + } + + failed := make(map[uint64]struct{}) + + log.Debugw("pruning headers", "from", headers[0].Height(), "to", + headers[len(headers)-1].Height()) + + for _, eh := range headers { + pruneCtx, cancel := context.WithTimeout(ctx, time.Second*5) + + err = s.pruner.Prune(pruneCtx, eh) + if err != nil { + log.Errorw("failed to prune block", "height", eh.Height(), "err", err) + failed[eh.Height()] = struct{}{} + } else { + lastPrunedHeader = eh + } + + s.metrics.observePrune(pruneCtx, err != nil) + cancel() + } + + err = s.updateCheckpoint(s.ctx, lastPrunedHeader.Height(), failed) + if err != nil { + log.Errorw("failed to update checkpoint", "err", err) + return lastPrunedHeader + } + + if uint64(len(headers)) < maxHeadersPerLoop { + // we've pruned all the blocks we can + return lastPrunedHeader + } + } +} + +func (s *Service) retryFailed(ctx context.Context) { + log.Debugw("retrying failed headers", "amount", len(s.checkpoint.FailedHeaders)) + + for failed := range s.checkpoint.FailedHeaders { + h, err := s.getter.GetByHeight(ctx, failed) + if err != nil { + log.Errorw("failed to load header from failed map", "height", failed, "err", err) + continue + } + err = s.pruner.Prune(ctx, h) + if err != nil { + log.Errorw("failed to prune block from failed map", "height", failed, "err", err) + continue + } + delete(s.checkpoint.FailedHeaders, failed) + } } diff --git a/pruner/service_test.go b/pruner/service_test.go new file mode 100644 index 0000000000..01932abaf2 --- /dev/null +++ b/pruner/service_test.go @@ -0,0 +1,349 @@ +package pruner + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/sync" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/header/headertest" +) + +/* + | toPrune | availability window | +*/ + +// TestService tests the pruner service to check whether the expected +// amount of blocks are pruned within a given AvailabilityWindow. +// This test runs a pruning cycle once which should prune at least +// 2 blocks (as the AvailabilityWindow is ~2 blocks). Since the +// prune-able header determination is time-based, it cannot be +// exact. +func TestService(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + blockTime := time.Millisecond + + // all headers generated in suite are timestamped to time.Now(), so + // they will all be considered "pruneable" within the availability window ( + suite := headertest.NewTestSuite(t, 1, blockTime) + store := headertest.NewCustomStore(t, suite, 20) + + mp := &mockPruner{} + + serv, err := NewService( + mp, + AvailabilityWindow(time.Millisecond*2), + store, + sync.MutexWrap(datastore.NewMapDatastore()), + blockTime, + ) + require.NoError(t, err) + + serv.ctx, serv.cancel = ctx, cancel + + err = serv.loadCheckpoint(ctx) + require.NoError(t, err) + + time.Sleep(time.Millisecond * 2) + + lastPruned, err := serv.lastPruned(ctx) + require.NoError(t, err) + lastPruned = serv.prune(ctx, lastPruned) + + assert.Greater(t, lastPruned.Height(), uint64(2)) + assert.Greater(t, serv.checkpoint.LastPrunedHeight, uint64(2)) +} + +// TestService_FailedAreRecorded checks whether the pruner service +// can accurately detect blocks to be pruned and store them +// to checkpoint. +func TestService_FailedAreRecorded(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + blockTime := time.Millisecond + + // all headers generated in suite are timestamped to time.Now(), so + // they will all be considered "pruneable" within the availability window + suite := headertest.NewTestSuite(t, 1, blockTime) + store := headertest.NewCustomStore(t, suite, 100) + + mp := &mockPruner{ + failHeight: map[uint64]int{4: 0, 5: 0, 13: 0}, + } + + serv, err := NewService( + mp, + AvailabilityWindow(time.Millisecond*20), + store, + sync.MutexWrap(datastore.NewMapDatastore()), + blockTime, + ) + require.NoError(t, err) + + serv.ctx = ctx + + err = serv.loadCheckpoint(ctx) + require.NoError(t, err) + + // ensures at least 13 blocks are prune-able + time.Sleep(time.Millisecond * 50) + + // trigger a prune job + lastPruned, err := serv.lastPruned(ctx) + require.NoError(t, err) + _ = serv.prune(ctx, lastPruned) + + assert.Len(t, serv.checkpoint.FailedHeaders, 3) + for expectedFail := range mp.failHeight { + _, exists := serv.checkpoint.FailedHeaders[expectedFail] + assert.True(t, exists) + } + + // trigger another prune job, which will prioritize retrying + // failed blocks + lastPruned, err = serv.lastPruned(ctx) + require.NoError(t, err) + _ = serv.prune(ctx, lastPruned) + + assert.Len(t, serv.checkpoint.FailedHeaders, 0) +} + +func TestServiceCheckpointing(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + store := headertest.NewStore(t) + + mp := &mockPruner{} + + serv, err := NewService( + mp, + AvailabilityWindow(time.Second), + store, + sync.MutexWrap(datastore.NewMapDatastore()), + time.Millisecond, + ) + require.NoError(t, err) + + err = serv.loadCheckpoint(ctx) + require.NoError(t, err) + + // ensure checkpoint was initialized correctly + assert.Equal(t, uint64(1), serv.checkpoint.LastPrunedHeight) + assert.Empty(t, serv.checkpoint.FailedHeaders) + + // update checkpoint + err = serv.updateCheckpoint(ctx, uint64(3), map[uint64]struct{}{2: {}}) + require.NoError(t, err) + + // ensure checkpoint was updated correctly in datastore + err = serv.loadCheckpoint(ctx) + require.NoError(t, err) + assert.Equal(t, uint64(3), serv.checkpoint.LastPrunedHeight) + assert.Len(t, serv.checkpoint.FailedHeaders, 1) +} + +// TestPrune_LargeNumberOfBlocks tests that the pruner service with a large +// number of blocks to prune (an archival node turning into a pruned node) is +// able to prune the blocks in one prune cycle. +func TestPrune_LargeNumberOfBlocks(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + maxHeadersPerLoop = 10 + t.Cleanup(func() { + maxHeadersPerLoop = 1024 + }) + + blockTime := time.Nanosecond + availabilityWindow := AvailabilityWindow(blockTime * 10) + + // all headers generated in suite are timestamped to time.Now(), so + // they will all be considered "pruneable" within the availability window + suite := headertest.NewTestSuite(t, 1, blockTime) + store := headertest.NewCustomStore(t, suite, int(maxHeadersPerLoop*6)) // add small buffer + + mp := &mockPruner{failHeight: make(map[uint64]int, 0)} + + serv, err := NewService( + mp, + availabilityWindow, + store, + sync.MutexWrap(datastore.NewMapDatastore()), + blockTime, + ) + require.NoError(t, err) + serv.ctx = ctx + + err = serv.loadCheckpoint(ctx) + require.NoError(t, err) + + // ensures availability window has passed + time.Sleep(time.Duration(availabilityWindow) + time.Millisecond*100) + + // trigger a prune job + lastPruned, err := serv.lastPruned(ctx) + require.NoError(t, err) + _ = serv.prune(ctx, lastPruned) + + // ensure all headers have been pruned + assert.Equal(t, maxHeadersPerLoop*5, serv.checkpoint.LastPrunedHeight) + assert.Len(t, serv.checkpoint.FailedHeaders, 0) +} + +func TestFindPruneableHeaders(t *testing.T) { + testCases := []struct { + name string + availWindow AvailabilityWindow + blockTime time.Duration + startTime time.Time + headerAmount int + expectedLength int + }{ + { + name: "Estimated range matches expected", + // Availability window is one week + availWindow: AvailabilityWindow(time.Hour * 24 * 7), + blockTime: time.Hour, + // Make two weeks of headers + headerAmount: 2 * (24 * 7), + startTime: time.Now().Add(-2 * time.Hour * 24 * 7), + // One week of headers are pruneable + expectedLength: (24 * 7) + 1, + }, + { + name: "Estimated range not sufficient but finds the correct tail", + // Availability window is one week + availWindow: AvailabilityWindow(time.Hour * 24 * 7), + blockTime: time.Hour, + // Make three weeks of headers + headerAmount: 3 * (24 * 7), + startTime: time.Now().Add(-3 * time.Hour * 24 * 7), + // Two weeks of headers are pruneable + expectedLength: (2 * 24 * 7) + 1, + }, + { + name: "No pruneable headers", + // Availability window is two weeks + availWindow: AvailabilityWindow(2 * time.Hour * 24 * 7), + blockTime: time.Hour, + // Make one week of headers + headerAmount: 24 * 7, + startTime: time.Now().Add(-time.Hour * 24 * 7), + // No headers are pruneable + expectedLength: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + headerGenerator := NewSpacedHeaderGenerator(t, tc.startTime, tc.blockTime) + store := headertest.NewCustomStore(t, headerGenerator, tc.headerAmount) + + mp := &mockPruner{} + + serv, err := NewService( + mp, + tc.availWindow, + store, + sync.MutexWrap(datastore.NewMapDatastore()), + tc.blockTime, + ) + require.NoError(t, err) + + err = serv.Start(ctx) + require.NoError(t, err) + + lastPruned, err := serv.lastPruned(ctx) + require.NoError(t, err) + + pruneable, err := serv.findPruneableHeaders(ctx, lastPruned) + require.NoError(t, err) + require.Len(t, pruneable, tc.expectedLength) + + pruneableCutoff := time.Now().Add(-time.Duration(tc.availWindow)) + // All returned headers are older than the availability window + for _, h := range pruneable { + require.WithinRange(t, h.Time(), tc.startTime, pruneableCutoff) + } + + // The next header after the last pruneable header is too new to prune + if len(pruneable) != 0 { + lastPruneable := pruneable[len(pruneable)-1] + if lastPruneable.Height() != store.Height() { + firstUnpruneable, err := store.GetByHeight(ctx, lastPruneable.Height()+1) + require.NoError(t, err) + require.WithinRange(t, firstUnpruneable.Time(), pruneableCutoff, time.Now()) + } + } + }) + } +} + +type mockPruner struct { + deletedHeaderHashes []pruned + + // tells the mockPruner on which heights to fail + failHeight map[uint64]int +} + +type pruned struct { + hash string + height uint64 +} + +func (mp *mockPruner) Prune(_ context.Context, h *header.ExtendedHeader) error { + for fail := range mp.failHeight { + if h.Height() == fail { + // if retried, return successful + if mp.failHeight[fail] > 0 { + return nil + } + mp.failHeight[fail]++ + return fmt.Errorf("failed to prune") + } + } + mp.deletedHeaderHashes = append(mp.deletedHeaderHashes, pruned{hash: h.Hash().String(), height: h.Height()}) + return nil +} + +// TODO @renaynay @distractedm1nd: Deduplicate via headertest utility. +// https://github.com/celestiaorg/celestia-node/issues/3278. +type SpacedHeaderGenerator struct { + t *testing.T + TimeBetweenHeaders time.Duration + currentTime time.Time + currentHeight int64 +} + +func NewSpacedHeaderGenerator( + t *testing.T, startTime time.Time, timeBetweenHeaders time.Duration, +) *SpacedHeaderGenerator { + return &SpacedHeaderGenerator{ + t: t, + TimeBetweenHeaders: timeBetweenHeaders, + currentTime: startTime, + currentHeight: 1, + } +} + +func (shg *SpacedHeaderGenerator) NextHeader() *header.ExtendedHeader { + h := headertest.RandExtendedHeaderAtTimestamp(shg.t, shg.currentTime) + h.RawHeader.Height = shg.currentHeight + h.RawHeader.Time = shg.currentTime + shg.currentHeight++ + shg.currentTime = shg.currentTime.Add(shg.TimeBetweenHeaders) + return h +} diff --git a/share/availability/full/availability.go b/share/availability/full/availability.go index 4ea211cb1e..2137e4a2b7 100644 --- a/share/availability/full/availability.go +++ b/share/availability/full/availability.go @@ -5,15 +5,16 @@ import ( "errors" "fmt" - "github.com/filecoin-project/dagstore" logging "github.com/ipfs/go-log/v2" + "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/byzantine" - "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p/discovery" + "github.com/celestiaorg/celestia-node/share/store" ) var log = logging.Logger("share/full") @@ -22,16 +23,17 @@ var log = logging.Logger("share/full") // recovery technique. It is considered "full" because it is required // to download enough shares to fully reconstruct the data square. type ShareAvailability struct { - store *eds.Store + store *store.Store getter share.Getter - disc *discovery.Discovery + // TODO(@walldiss): discovery should be managed by nodebuilder, not availability + disc *discovery.Discovery cancel context.CancelFunc } // NewShareAvailability creates a new full ShareAvailability. func NewShareAvailability( - store *eds.Store, + store *store.Store, getter share.Getter, disc *discovery.Discovery, ) *ShareAvailability { @@ -58,45 +60,42 @@ func (fa *ShareAvailability) Stop(context.Context) error { // SharesAvailable reconstructs the data committed to the given Root by requesting // enough Shares from the network. func (fa *ShareAvailability) SharesAvailable(ctx context.Context, header *header.ExtendedHeader) error { - dah := header.DAH - // short-circuit if the given root is minimum DAH of an empty data square, to avoid datastore hit - if share.DataHash(dah.Hash()).IsEmptyRoot() { - return nil + // a hack to avoid loading the whole EDS in mem if we store it already. + if ok, _ := fa.store.HasByHash(ctx, header.DAH.Hash()); ok { + return fa.store.LinkHashToHeight(ctx, header.DAH.Hash(), header.Height()) } - // we assume the caller of this method has already performed basic validation on the - // given dah/root. If for some reason this has not happened, the node should panic. - if err := dah.ValidateBasic(); err != nil { - log.Errorw("Availability validation cannot be performed on a malformed DataAvailabilityHeader", - "err", err) - panic(err) + eds, err := fa.getEds(ctx, header) + if err != nil { + return err } - // a hack to avoid loading the whole EDS in mem if we store it already. - if ok, _ := fa.store.Has(ctx, dah.Hash()); ok { - return nil + f, err := fa.store.Put(ctx, header.DAH.Hash(), header.Height(), eds) + if err != nil { + return fmt.Errorf("full availability: failed to store eds: %w", err) } + utils.CloseAndLog(log, "file", f) + return nil +} - adder := ipld.NewProofsAdder(len(dah.RowRoots)) - ctx = ipld.CtxWithProofsAdder(ctx, adder) - defer adder.Purge() +func (fa *ShareAvailability) getEds(ctx context.Context, header *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { + dah := header.DAH + // short-circuit if the given root is minimum DAH of an empty data square, to avoid datastore hit + if share.DataHash(dah.Hash()).IsEmptyRoot() { + return share.EmptyExtendedDataSquare(), nil + } eds, err := fa.getter.GetEDS(ctx, header) if err != nil { if errors.Is(err, context.Canceled) { - return err + return nil, err } log.Errorw("availability validation failed", "root", dah.String(), "err", err.Error()) var byzantineErr *byzantine.ErrByzantine if errors.Is(err, share.ErrNotFound) || errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &byzantineErr) { - return share.ErrNotAvailable + return nil, fmt.Errorf("%w:%w", share.ErrNotAvailable, err) } - return err + return nil, err } - - err = fa.store.Put(ctx, dah.Hash(), eds) - if err != nil && !errors.Is(err, dagstore.ErrShardExists) { - return fmt.Errorf("full availability: failed to store eds: %w", err) - } - return nil + return eds, nil } diff --git a/share/availability/full/availability_test.go b/share/availability/full/availability_test.go index 8ac0648a87..ce5a240a2e 100644 --- a/share/availability/full/availability_test.go +++ b/share/availability/full/availability_test.go @@ -1,82 +1,83 @@ package full -import ( - "context" - "testing" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-app/pkg/da" - - "github.com/celestiaorg/celestia-node/header/headertest" - "github.com/celestiaorg/celestia-node/share" - availability_test "github.com/celestiaorg/celestia-node/share/availability/test" - "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/mocks" -) - -func TestShareAvailableOverMocknet_Full(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - net := availability_test.NewTestDAGNet(ctx, t) - _, root := RandNode(net, 32) - - eh := headertest.RandExtendedHeaderWithRoot(t, root) - nd := Node(net) - net.ConnectAll() - - err := nd.SharesAvailable(ctx, eh) - assert.NoError(t, err) -} - -func TestSharesAvailable_Full(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // RandServiceWithSquare creates a NewShareAvailability inside, so we can test it - getter, dah := GetterWithRandSquare(t, 16) - - eh := headertest.RandExtendedHeaderWithRoot(t, dah) - avail := TestAvailability(t, getter) - err := avail.SharesAvailable(ctx, eh) - assert.NoError(t, err) -} - -func TestSharesAvailable_StoresToEDSStore(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // RandServiceWithSquare creates a NewShareAvailability inside, so we can test it - getter, dah := GetterWithRandSquare(t, 16) - eh := headertest.RandExtendedHeaderWithRoot(t, dah) - avail := TestAvailability(t, getter) - err := avail.SharesAvailable(ctx, eh) - assert.NoError(t, err) - - has, err := avail.store.Has(ctx, dah.Hash()) - assert.NoError(t, err) - assert.True(t, has) -} - -func TestSharesAvailable_Full_ErrNotAvailable(t *testing.T) { - ctrl := gomock.NewController(t) - getter := mocks.NewMockGetter(ctrl) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - eds := edstest.RandEDS(t, 4) - dah, err := da.NewDataAvailabilityHeader(eds) - eh := headertest.RandExtendedHeaderWithRoot(t, &dah) - require.NoError(t, err) - avail := TestAvailability(t, getter) - - errors := []error{share.ErrNotFound, context.DeadlineExceeded} - for _, getterErr := range errors { - getter.EXPECT().GetEDS(gomock.Any(), gomock.Any()).Return(nil, getterErr) - err := avail.SharesAvailable(ctx, eh) - require.ErrorIs(t, err, share.ErrNotAvailable) - } -} +// TODO(@walldiss): rework all availability tests +//import ( +// "context" +// "testing" +// +// "github.com/golang/mock/gomock" +// "github.com/stretchr/testify/assert" +// "github.com/stretchr/testify/require" +// +// "github.com/celestiaorg/celestia-app/pkg/da" +// +// "github.com/celestiaorg/celestia-node/header/headertest" +// "github.com/celestiaorg/celestia-node/share" +// availability_test "github.com/celestiaorg/celestia-node/share/availability/test" +// "github.com/celestiaorg/celestia-node/share/testing/edstest" +// "github.com/celestiaorg/celestia-node/share/mocks" +//) +// +//func TestShareAvailableOverMocknet_Full(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// net := availability_test.NewTestDAGNet(ctx, t) +// _, root := RandNode(net, 32) +// +// eh := headertest.RandExtendedHeaderWithRoot(t, root) +// nd := Node(net) +// net.ConnectAll() +// +// err := nd.SharesAvailable(ctx, eh) +// assert.NoError(t, err) +//} +// +//func TestSharesAvailable_Full(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// // RandServiceWithSquare creates a NewShareAvailability inside, so we can test it +// getter, dah := GetterWithRandSquare(t, 16) +// +// eh := headertest.RandExtendedHeaderWithRoot(t, dah) +// avail := TestAvailability(t, getter) +// err := avail.SharesAvailable(ctx, eh) +// assert.NoError(t, err) +//} +// +//func TestSharesAvailable_StoresToEDSStore(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// // RandServiceWithSquare creates a NewShareAvailability inside, so we can test it +// getter, dah := GetterWithRandSquare(t, 16) +// eh := headertest.RandExtendedHeaderWithRoot(t, dah) +// avail := TestAvailability(t, getter) +// err := avail.SharesAvailable(ctx, eh) +// assert.NoError(t, err) +// +// has, err := avail.store.Has(ctx, dah.Hash()) +// assert.NoError(t, err) +// assert.True(t, has) +//} +// +//func TestSharesAvailable_Full_ErrNotAvailable(t *testing.T) { +// ctrl := gomock.NewController(t) +// getter := mocks.NewMockGetter(ctrl) +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// eds := edstest.RandEDS(t, 4) +// dah, err := da.NewDataAvailabilityHeader(eds) +// eh := headertest.RandExtendedHeaderWithRoot(t, &dah) +// require.NoError(t, err) +// avail := TestAvailability(t, getter) +// +// errors := []error{share.ErrNotFound, context.DeadlineExceeded} +// for _, getterErr := range errors { +// getter.EXPECT().GetEDS(gomock.Any(), gomock.Any()).Return(nil, getterErr) +// err := avail.SharesAvailable(ctx, eh) +// require.ErrorIs(t, err, share.ErrNotAvailable) +// } +//} diff --git a/share/availability/full/testing.go b/share/availability/full/testing.go index 46e97581f2..0dfb167e01 100644 --- a/share/availability/full/testing.go +++ b/share/availability/full/testing.go @@ -1,64 +1,47 @@ package full -import ( - "context" - "testing" - "time" - - "github.com/ipfs/go-datastore" - routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" - "github.com/libp2p/go-libp2p/p2p/discovery/routing" - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-node/share" - availability_test "github.com/celestiaorg/celestia-node/share/availability/test" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/getters" - "github.com/celestiaorg/celestia-node/share/ipld" - "github.com/celestiaorg/celestia-node/share/p2p/discovery" -) - +// FIXME: rework testing pkg // GetterWithRandSquare provides a share.Getter filled with 'n' NMT // trees of 'n' random shares, essentially storing a whole square. -func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *share.Root) { - bServ := ipld.NewMemBlockservice() - getter := getters.NewIPLDGetter(bServ) - return getter, availability_test.RandFillBS(t, n, bServ) -} - -// RandNode creates a Full Node filled with a random block of the given size. -func RandNode(dn *availability_test.TestDagNet, squareSize int) (*availability_test.TestNode, *share.Root) { - nd := Node(dn) - return nd, availability_test.RandFillBS(dn.T, squareSize, nd.BlockService) -} - -// Node creates a new empty Full Node. -func Node(dn *availability_test.TestDagNet) *availability_test.TestNode { - nd := dn.NewTestNode() - nd.Getter = getters.NewIPLDGetter(nd.BlockService) - nd.Availability = TestAvailability(dn.T, nd.Getter) - return nd -} - -func TestAvailability(t *testing.T, getter share.Getter) *ShareAvailability { - params := discovery.DefaultParameters() - params.AdvertiseInterval = time.Second - params.PeersLimit = 10 - disc, err := discovery.NewDiscovery( - params, - nil, - routing.NewRoutingDiscovery(routinghelpers.Null{}), - "full", - ) - require.NoError(t, err) - store, err := eds.NewStore(eds.DefaultParameters(), t.TempDir(), datastore.NewMapDatastore()) - require.NoError(t, err) - err = store.Start(context.Background()) - require.NoError(t, err) - - t.Cleanup(func() { - err = store.Stop(context.Background()) - require.NoError(t, err) - }) - return NewShareAvailability(store, getter, disc) -} +//func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *share.Root) { +// bServ := ipld.NewMemBlockservice() +// getter := getters.NewIPLDGetter(bServ) +// return getter, availability_test.RandFillBS(t, n, bServ) +//} +// +//// RandNode creates a Full Node filled with a random block of the given size. +//func RandNode(dn *availability_test.TestDagNet, squareSize int) (*availability_test.TestNode, *share.Root) { +// nd := Node(dn) +// return nd, availability_test.RandFillBS(dn.T, squareSize, nd.BlockService) +//} +// +//// Node creates a new empty Full Node. +//func Node(dn *availability_test.TestDagNet) *availability_test.TestNode { +// nd := dn.NewTestNode() +// nd.Getter = getters.NewIPLDGetter(nd.BlockService) +// nd.Availability = TestAvailability(dn.T, nd.Getter) +// return nd +//} +// +//func TestAvailability(t *testing.T, getter share.Getter) *ShareAvailability { +// params := discovery.DefaultParameters() +// params.AdvertiseInterval = time.Second +// params.PeersLimit = 10 +// disc, err := discovery.NewDiscovery( +// params, +// nil, +// routing.NewRoutingDiscovery(routinghelpers.Null{}), +// "full", +// ) +// require.NoError(t, err) +// store, err := eds.NewStore(eds.DefaultParameters(), t.TempDir(), datastore.NewMapDatastore()) +// require.NoError(t, err) +// err = store.Start(context.Background()) +// require.NoError(t, err) +// +// t.Cleanup(func() { +// err = store.Stop(context.Background()) +// require.NoError(t, err) +// }) +// return NewShareAvailability(store, getter, disc) +//} diff --git a/share/availability/light/availability.go b/share/availability/light/availability.go index 97046f4438..f2badf6796 100644 --- a/share/availability/light/availability.go +++ b/share/availability/light/availability.go @@ -12,7 +12,6 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/getters" ) var ( @@ -100,10 +99,6 @@ func (la *ShareAvailability) SharesAvailable(ctx context.Context, header *header return err } - // indicate to the share.Getter that a blockservice session should be created. This - // functionality is optional and must be supported by the used share.Getter. - ctx = getters.WithSession(ctx) - var ( failedSamplesLock sync.Mutex failedSamples []Sample diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index 68da3698b5..976de24488 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -1,257 +1,258 @@ package light -import ( - "context" - _ "embed" - "strconv" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-node/header/headertest" - "github.com/celestiaorg/celestia-node/share" - availability_test "github.com/celestiaorg/celestia-node/share/availability/test" - "github.com/celestiaorg/celestia-node/share/ipld" - "github.com/celestiaorg/celestia-node/share/sharetest" -) - -func TestSharesAvailableCaches(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - getter, eh := GetterWithRandSquare(t, 16) - dah := eh.DAH - avail := TestAvailability(getter) - - // cache doesn't have dah yet - has, err := avail.ds.Has(ctx, rootKey(dah)) - require.NoError(t, err) - require.False(t, has) - - err = avail.SharesAvailable(ctx, eh) - require.NoError(t, err) - - // is now stored success result - result, err := avail.ds.Get(ctx, rootKey(dah)) - require.NoError(t, err) - failed, err := decodeSamples(result) - require.NoError(t, err) - require.Empty(t, failed) -} - -func TestSharesAvailableHitsCache(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - getter, _ := GetterWithRandSquare(t, 16) - avail := TestAvailability(getter) - - // create new dah, that is not available by getter - bServ := ipld.NewMemBlockservice() - dah := availability_test.RandFillBS(t, 16, bServ) - eh := headertest.RandExtendedHeaderWithRoot(t, dah) - - // blockstore doesn't actually have the dah - err := avail.SharesAvailable(ctx, eh) - require.ErrorIs(t, err, share.ErrNotAvailable) - - // put success result in cache - err = avail.ds.Put(ctx, rootKey(dah), []byte{}) - require.NoError(t, err) - - // should hit cache after putting - err = avail.SharesAvailable(ctx, eh) - require.NoError(t, err) -} - -func TestSharesAvailableEmptyRoot(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - getter, _ := GetterWithRandSquare(t, 16) - avail := TestAvailability(getter) - - eh := headertest.RandExtendedHeaderWithRoot(t, share.EmptyRoot()) - err := avail.SharesAvailable(ctx, eh) - require.NoError(t, err) -} - -func TestSharesAvailableFailed(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - getter, _ := GetterWithRandSquare(t, 16) - avail := TestAvailability(getter) - - // create new dah, that is not available by getter - bServ := ipld.NewMemBlockservice() - dah := availability_test.RandFillBS(t, 16, bServ) - eh := headertest.RandExtendedHeaderWithRoot(t, dah) - - // blockstore doesn't actually have the dah, so it should fail - err := avail.SharesAvailable(ctx, eh) - require.ErrorIs(t, err, share.ErrNotAvailable) - - // cache should have failed results now - result, err := avail.ds.Get(ctx, rootKey(dah)) - require.NoError(t, err) - - failed, err := decodeSamples(result) - require.NoError(t, err) - require.Len(t, failed, int(avail.params.SampleAmount)) - - // ensure that retry persists the failed samples selection - // create new getter with only the failed samples available, and add them to the onceGetter - onceGetter := newOnceGetter() - onceGetter.AddSamples(failed) - - // replace getter with the new one - avail.getter = onceGetter - - // should be able to retrieve all the failed samples now - err = avail.SharesAvailable(ctx, eh) - require.NoError(t, err) - - // onceGetter should have no more samples stored after the call - require.Empty(t, onceGetter.available) -} - -func TestShareAvailableOverMocknet_Light(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - net := availability_test.NewTestDAGNet(ctx, t) - _, root := RandNode(net, 16) - eh := headertest.RandExtendedHeader(t) - eh.DAH = root - nd := Node(net) - net.ConnectAll() - - err := nd.SharesAvailable(ctx, eh) - require.NoError(t, err) -} - -func TestGetShare(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - n := 16 - getter, eh := GetterWithRandSquare(t, n) - - for i := range make([]bool, n) { - for j := range make([]bool, n) { - sh, err := getter.GetShare(ctx, eh, i, j) - require.NotNil(t, sh) - require.NoError(t, err) - } - } -} - -func TestService_GetSharesByNamespace(t *testing.T) { - var tests = []struct { - squareSize int - expectedShareCount int - }{ - {squareSize: 4, expectedShareCount: 2}, - {squareSize: 16, expectedShareCount: 2}, - {squareSize: 128, expectedShareCount: 2}, - } - - for _, tt := range tests { - t.Run("size: "+strconv.Itoa(tt.squareSize), func(t *testing.T) { - getter, bServ := EmptyGetter() - totalShares := tt.squareSize * tt.squareSize - randShares := sharetest.RandShares(t, totalShares) - idx1 := (totalShares - 1) / 2 - idx2 := totalShares / 2 - if tt.expectedShareCount > 1 { - // make it so that two rows have the same namespace - copy(share.GetNamespace(randShares[idx2]), share.GetNamespace(randShares[idx1])) - } - root := availability_test.FillBS(t, bServ, randShares) - eh := headertest.RandExtendedHeader(t) - eh.DAH = root - randNamespace := share.GetNamespace(randShares[idx1]) - - shares, err := getter.GetSharesByNamespace(context.Background(), eh, randNamespace) - require.NoError(t, err) - require.NoError(t, shares.Verify(root, randNamespace)) - flattened := shares.Flatten() - require.Len(t, flattened, tt.expectedShareCount) - for _, value := range flattened { - require.Equal(t, randNamespace, share.GetNamespace(value)) - } - if tt.expectedShareCount > 1 { - // idx1 is always smaller than idx2 - require.Equal(t, randShares[idx1], flattened[0]) - require.Equal(t, randShares[idx2], flattened[1]) - } - }) - t.Run("last two rows of a 4x4 square that have the same namespace have valid NMT proofs", func(t *testing.T) { - squareSize := 4 - totalShares := squareSize * squareSize - getter, bServ := EmptyGetter() - randShares := sharetest.RandShares(t, totalShares) - lastNID := share.GetNamespace(randShares[totalShares-1]) - for i := totalShares / 2; i < totalShares; i++ { - copy(share.GetNamespace(randShares[i]), lastNID) - } - root := availability_test.FillBS(t, bServ, randShares) - eh := headertest.RandExtendedHeader(t) - eh.DAH = root - - shares, err := getter.GetSharesByNamespace(context.Background(), eh, lastNID) - require.NoError(t, err) - require.NoError(t, shares.Verify(root, lastNID)) - }) - } -} - -func TestGetShares(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - n := 16 - getter, eh := GetterWithRandSquare(t, n) - - eds, err := getter.GetEDS(ctx, eh) - require.NoError(t, err) - gotDAH, err := share.NewRoot(eds) - require.NoError(t, err) - - require.True(t, eh.DAH.Equals(gotDAH)) -} - -func TestService_GetSharesByNamespaceNotFound(t *testing.T) { - getter, eh := GetterWithRandSquare(t, 1) - eh.DAH.RowRoots = nil - - emptyShares, err := getter.GetSharesByNamespace(context.Background(), eh, sharetest.RandV0Namespace()) - require.NoError(t, err) - require.Empty(t, emptyShares.Flatten()) -} - -func BenchmarkService_GetSharesByNamespace(b *testing.B) { - var tests = []struct { - amountShares int - }{ - {amountShares: 4}, - {amountShares: 16}, - {amountShares: 128}, - } - - for _, tt := range tests { - b.Run(strconv.Itoa(tt.amountShares), func(b *testing.B) { - t := &testing.T{} - getter, eh := GetterWithRandSquare(t, tt.amountShares) - root := eh.DAH - randNamespace := root.RowRoots[(len(root.RowRoots)-1)/2][:share.NamespaceSize] - root.RowRoots[(len(root.RowRoots) / 2)] = root.RowRoots[(len(root.RowRoots)-1)/2] - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, err := getter.GetSharesByNamespace(context.Background(), eh, randNamespace) - require.NoError(t, err) - } - }) - } -} +// TODO(@walldiss): rework all availability tests +//import ( +// "context" +// _ "embed" +// "strconv" +// "testing" +// +// "github.com/stretchr/testify/require" +// +// "github.com/celestiaorg/celestia-node/header/headertest" +// "github.com/celestiaorg/celestia-node/share" +// availability_test "github.com/celestiaorg/celestia-node/share/availability/test" +// "github.com/celestiaorg/celestia-node/share/ipld" +// "github.com/celestiaorg/celestia-node/share/sharetest" +//) +// +//func TestSharesAvailableCaches(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// getter, eh := GetterWithRandSquare(t, 16) +// dah := eh.DAH +// avail := TestAvailability(getter) +// +// // cache doesn't have dah yet +// has, err := avail.ds.Has(ctx, rootKey(dah)) +// require.NoError(t, err) +// require.False(t, has) +// +// err = avail.SharesAvailable(ctx, eh) +// require.NoError(t, err) +// +// // is now stored success result +// result, err := avail.ds.Get(ctx, rootKey(dah)) +// require.NoError(t, err) +// failed, err := decodeSamples(result) +// require.NoError(t, err) +// require.Empty(t, failed) +//} +// +//func TestSharesAvailableHitsCache(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// getter, _ := GetterWithRandSquare(t, 16) +// avail := TestAvailability(getter) +// +// // create new dah, that is not available by getter +// bServ := ipld.NewMemBlockservice() +// dah := availability_test.RandFillBS(t, 16, bServ) +// eh := headertest.RandExtendedHeaderWithRoot(t, dah) +// +// // blockstore doesn't actually have the dah +// err := avail.SharesAvailable(ctx, eh) +// require.ErrorIs(t, err, share.ErrNotAvailable) +// +// // put success result in cache +// err = avail.ds.Put(ctx, rootKey(dah), []byte{}) +// require.NoError(t, err) +// +// // should hit cache after putting +// err = avail.SharesAvailable(ctx, eh) +// require.NoError(t, err) +//} +// +//func TestSharesAvailableEmptyRoot(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// getter, _ := GetterWithRandSquare(t, 16) +// avail := TestAvailability(getter) +// +// eh := headertest.RandExtendedHeaderWithRoot(t, share.EmptyRoot()) +// err := avail.SharesAvailable(ctx, eh) +// require.NoError(t, err) +//} +// +//func TestSharesAvailableFailed(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// getter, _ := GetterWithRandSquare(t, 16) +// avail := TestAvailability(getter) +// +// // create new dah, that is not available by getter +// bServ := ipld.NewMemBlockservice() +// dah := availability_test.RandFillBS(t, 16, bServ) +// eh := headertest.RandExtendedHeaderWithRoot(t, dah) +// +// // blockstore doesn't actually have the dah, so it should fail +// err := avail.SharesAvailable(ctx, eh) +// require.ErrorIs(t, err, share.ErrNotAvailable) +// +// // cache should have failed results now +// result, err := avail.ds.Get(ctx, rootKey(dah)) +// require.NoError(t, err) +// +// failed, err := decodeSamples(result) +// require.NoError(t, err) +// require.Len(t, failed, int(avail.params.SampleAmount)) +// +// // ensure that retry persists the failed samples selection +// // create new getter with only the failed samples available, and add them to the onceGetter +// onceGetter := newOnceGetter() +// onceGetter.AddSamples(failed) +// +// // replace getter with the new one +// avail.getter = onceGetter +// +// // should be able to retrieve all the failed samples now +// err = avail.SharesAvailable(ctx, eh) +// require.NoError(t, err) +// +// // onceGetter should have no more samples stored after the call +// require.Empty(t, onceGetter.available) +//} +// +//func TestShareAvailableOverMocknet_Light(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// net := availability_test.NewTestDAGNet(ctx, t) +// _, root := RandNode(net, 16) +// eh := headertest.RandExtendedHeader(t) +// eh.DAH = root +// nd := Node(net) +// net.ConnectAll() +// +// err := nd.SharesAvailable(ctx, eh) +// require.NoError(t, err) +//} +// +//func TestGetShare(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// n := 16 +// getter, eh := GetterWithRandSquare(t, n) +// +// for i := range make([]bool, n) { +// for j := range make([]bool, n) { +// sh, err := getter.GetShare(ctx, eh, i, j) +// require.NotNil(t, sh) +// require.NoError(t, err) +// } +// } +//} +// +//func TestService_GetSharesByNamespace(t *testing.T) { +// var tests = []struct { +// squareSize int +// expectedShareCount int +// }{ +// {squareSize: 4, expectedShareCount: 2}, +// {squareSize: 16, expectedShareCount: 2}, +// {squareSize: 128, expectedShareCount: 2}, +// } +// +// for _, tt := range tests { +// t.Run("size: "+strconv.Itoa(tt.squareSize), func(t *testing.T) { +// getter, bServ := EmptyGetter() +// totalShares := tt.squareSize * tt.squareSize +// randShares := sharetest.RandShares(t, totalShares) +// idx1 := (totalShares - 1) / 2 +// idx2 := totalShares / 2 +// if tt.expectedShareCount > 1 { +// // make it so that two rows have the same namespace +// copy(share.GetNamespace(randShares[idx2]), share.GetNamespace(randShares[idx1])) +// } +// root := availability_test.FillBS(t, bServ, randShares) +// eh := headertest.RandExtendedHeader(t) +// eh.DAH = root +// randNamespace := share.GetNamespace(randShares[idx1]) +// +// shares, err := getter.GetSharesByNamespace(context.Background(), eh, randNamespace) +// require.NoError(t, err) +// require.NoError(t, shares.Verify(root, randNamespace)) +// flattened := shares.Flatten() +// require.Len(t, flattened, tt.expectedShareCount) +// for _, value := range flattened { +// require.Equal(t, randNamespace, share.GetNamespace(value)) +// } +// if tt.expectedShareCount > 1 { +// // idx1 is always smaller than idx2 +// require.Equal(t, randShares[idx1], flattened[0]) +// require.Equal(t, randShares[idx2], flattened[1]) +// } +// }) +// t.Run("last two rows of a 4x4 square that have the same namespace have valid NMT proofs", func(t *testing.T) { +// squareSize := 4 +// totalShares := squareSize * squareSize +// getter, bServ := EmptyGetter() +// randShares := sharetest.RandShares(t, totalShares) +// lastNID := share.GetNamespace(randShares[totalShares-1]) +// for i := totalShares / 2; i < totalShares; i++ { +// copy(share.GetNamespace(randShares[i]), lastNID) +// } +// root := availability_test.FillBS(t, bServ, randShares) +// eh := headertest.RandExtendedHeader(t) +// eh.DAH = root +// +// shares, err := getter.GetSharesByNamespace(context.Background(), eh, lastNID) +// require.NoError(t, err) +// require.NoError(t, shares.Verify(root, lastNID)) +// }) +// } +//} +// +//func TestGetShares(t *testing.T) { +// ctx, cancel := context.WithCancel(context.Background()) +// defer cancel() +// +// n := 16 +// getter, eh := GetterWithRandSquare(t, n) +// +// eds, err := getter.GetEDS(ctx, eh) +// require.NoError(t, err) +// gotDAH, err := share.NewRoot(eds) +// require.NoError(t, err) +// +// require.True(t, eh.DAH.Equals(gotDAH)) +//} +// +//func TestService_GetSharesByNamespaceNotFound(t *testing.T) { +// getter, eh := GetterWithRandSquare(t, 1) +// eh.DAH.RowRoots = nil +// +// emptyShares, err := getter.GetSharesByNamespace(context.Background(), eh, sharetest.RandV0Namespace()) +// require.NoError(t, err) +// require.Empty(t, emptyShares.Flatten()) +//} +// +//func BenchmarkService_GetSharesByNamespace(b *testing.B) { +// var tests = []struct { +// amountShares int +// }{ +// {amountShares: 4}, +// {amountShares: 16}, +// {amountShares: 128}, +// } +// +// for _, tt := range tests { +// b.Run(strconv.Itoa(tt.amountShares), func(b *testing.B) { +// t := &testing.T{} +// getter, eh := GetterWithRandSquare(t, tt.amountShares) +// root := eh.DAH +// randNamespace := root.RowRoots[(len(root.RowRoots)-1)/2][:share.NamespaceSize] +// root.RowRoots[(len(root.RowRoots) / 2)] = root.RowRoots[(len(root.RowRoots)-1)/2] +// b.ResetTimer() +// for i := 0; i < b.N; i++ { +// _, err := getter.GetSharesByNamespace(context.Background(), eh, randNamespace) +// require.NoError(t, err) +// } +// }) +// } +//} diff --git a/share/availability/light/testing.go b/share/availability/light/testing.go index b6251b4fbd..c81422db94 100644 --- a/share/availability/light/testing.go +++ b/share/availability/light/testing.go @@ -1,107 +1,108 @@ package light -import ( - "context" - "sync" - "testing" - - "github.com/ipfs/boxo/blockservice" - "github.com/ipfs/go-datastore" - - "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/header/headertest" - "github.com/celestiaorg/celestia-node/share" - availability_test "github.com/celestiaorg/celestia-node/share/availability/test" - "github.com/celestiaorg/celestia-node/share/getters" - "github.com/celestiaorg/celestia-node/share/ipld" -) - -// GetterWithRandSquare provides a share.Getter filled with 'n' NMT trees of 'n' random shares, -// essentially storing a whole square. -func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *header.ExtendedHeader) { - bServ := ipld.NewMemBlockservice() - getter := getters.NewIPLDGetter(bServ) - root := availability_test.RandFillBS(t, n, bServ) - eh := headertest.RandExtendedHeader(t) - eh.DAH = root - - return getter, eh -} - -// EmptyGetter provides an unfilled share.Getter with corresponding blockservice.BlockService than -// can be filled by the test. -func EmptyGetter() (share.Getter, blockservice.BlockService) { - bServ := ipld.NewMemBlockservice() - getter := getters.NewIPLDGetter(bServ) - return getter, bServ -} - -// RandNode creates a Light Node filled with a random block of the given size. -func RandNode(dn *availability_test.TestDagNet, squareSize int) (*availability_test.TestNode, *share.Root) { - nd := Node(dn) - return nd, availability_test.RandFillBS(dn.T, squareSize, nd.BlockService) -} - -// Node creates a new empty Light Node. -func Node(dn *availability_test.TestDagNet) *availability_test.TestNode { - nd := dn.NewTestNode() - nd.Getter = getters.NewIPLDGetter(nd.BlockService) - nd.Availability = TestAvailability(nd.Getter) - return nd -} - -func TestAvailability(getter share.Getter) *ShareAvailability { - ds := datastore.NewMapDatastore() - return NewShareAvailability(getter, ds) -} - -func SubNetNode(sn *availability_test.SubNet) *availability_test.TestNode { - nd := Node(sn.TestDagNet) - sn.AddNode(nd) - return nd -} - -type onceGetter struct { - *sync.Mutex - available map[Sample]struct{} -} - -func newOnceGetter() onceGetter { - return onceGetter{ - Mutex: &sync.Mutex{}, - available: make(map[Sample]struct{}), - } -} - -func (m onceGetter) AddSamples(samples []Sample) { - m.Lock() - defer m.Unlock() - for _, s := range samples { - m.available[s] = struct{}{} - } -} - -func (m onceGetter) GetShare(_ context.Context, _ *header.ExtendedHeader, row, col int) (share.Share, error) { - m.Lock() - defer m.Unlock() - s := Sample{Row: uint16(row), Col: uint16(col)} - if _, ok := m.available[s]; ok { - delete(m.available, s) - return share.Share{}, nil - } - return share.Share{}, share.ErrNotAvailable -} - -func (m onceGetter) GetEDS(_ context.Context, _ *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { - panic("not implemented") -} - -func (m onceGetter) GetSharesByNamespace( - _ context.Context, - _ *header.ExtendedHeader, - _ share.Namespace, -) (share.NamespacedShares, error) { - panic("not implemented") -} +// TODO(@walldiss): rework all availability tests +//import ( +// "context" +// "sync" +// "testing" +// +// "github.com/ipfs/boxo/blockservice" +// "github.com/ipfs/go-datastore" +// +// "github.com/celestiaorg/rsmt2d" +// +// "github.com/celestiaorg/celestia-node/header" +// "github.com/celestiaorg/celestia-node/header/headertest" +// "github.com/celestiaorg/celestia-node/share" +// availability_test "github.com/celestiaorg/celestia-node/share/availability/test" +// "github.com/celestiaorg/celestia-node/share/getters" +// "github.com/celestiaorg/celestia-node/share/ipld" +//) +// +//// GetterWithRandSquare provides a share.Getter filled with 'n' NMT trees of 'n' random shares, +//// essentially storing a whole square. +//func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *header.ExtendedHeader) { +// bServ := ipld.NewMemBlockservice() +// getter := getters.NewIPLDGetter(bServ) +// root := availability_test.RandFillBS(t, n, bServ) +// eh := headertest.RandExtendedHeader(t) +// eh.DAH = root +// +// return getter, eh +//} +// +//// EmptyGetter provides an unfilled share.Getter with corresponding blockservice.BlockService than +//// can be filled by the test. +//func EmptyGetter() (share.Getter, blockservice.BlockService) { +// bServ := ipld.NewMemBlockservice() +// getter := getters.NewIPLDGetter(bServ) +// return getter, bServ +//} +// +//// RandNode creates a Light Node filled with a random block of the given size. +//func RandNode(dn *availability_test.TestDagNet, squareSize int) (*availability_test.TestNode, *share.Root) { +// nd := Node(dn) +// return nd, availability_test.RandFillBS(dn.T, squareSize, nd.BlockService) +//} +// +//// Node creates a new empty Light Node. +//func Node(dn *availability_test.TestDagNet) *availability_test.TestNode { +// nd := dn.NewTestNode() +// nd.Getter = getters.NewIPLDGetter(nd.BlockService) +// nd.Availability = TestAvailability(nd.Getter) +// return nd +//} +// +//func TestAvailability(getter share.Getter) *ShareAvailability { +// ds := datastore.NewMapDatastore() +// return NewShareAvailability(getter, ds) +//} +// +//func SubNetNode(sn *availability_test.SubNet) *availability_test.TestNode { +// nd := Node(sn.TestDagNet) +// sn.AddNode(nd) +// return nd +//} +// +//type onceGetter struct { +// *sync.Mutex +// available map[Sample]struct{} +//} +// +//func newOnceGetter() onceGetter { +// return onceGetter{ +// Mutex: &sync.Mutex{}, +// available: make(map[Sample]struct{}), +// } +//} +// +//func (m onceGetter) AddSamples(samples []Sample) { +// m.Lock() +// defer m.Unlock() +// for _, s := range samples { +// m.available[s] = struct{}{} +// } +//} +// +//func (m onceGetter) GetShare(_ context.Context, _ *header.ExtendedHeader, row, col int) (share.Share, error) { +// m.Lock() +// defer m.Unlock() +// s := Sample{Row: uint16(row), Col: uint16(col)} +// if _, ok := m.available[s]; ok { +// delete(m.available, s) +// return share.Share{}, nil +// } +// return share.Share{}, share.ErrNotAvailable +//} +// +//func (m onceGetter) GetEDS(_ context.Context, _ *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { +// panic("not implemented") +//} +// +//func (m onceGetter) GetSharesByNamespace( +// _ context.Context, +// _ *header.ExtendedHeader, +// _ share.Namespace, +//) (share.NamespacedShares, error) { +// panic("not implemented") +//} diff --git a/share/availability/test/testing.go b/share/availability/test/testing.go index 64e8d23bb7..8ac0a8d03a 100644 --- a/share/availability/test/testing.go +++ b/share/availability/test/testing.go @@ -19,7 +19,7 @@ import ( "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" - "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" ) // RandFillBS fills the given BlockService with a random block of a given size. diff --git a/share/eds/adapters.go b/share/eds/adapters.go deleted file mode 100644 index 8bf2340d91..0000000000 --- a/share/eds/adapters.go +++ /dev/null @@ -1,66 +0,0 @@ -package eds - -import ( - "context" - "sync" - - "github.com/filecoin-project/dagstore" - "github.com/ipfs/boxo/blockservice" - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" -) - -var _ blockservice.BlockGetter = (*BlockGetter)(nil) - -// NewBlockGetter creates new blockservice.BlockGetter adapter from dagstore.ReadBlockstore -func NewBlockGetter(store dagstore.ReadBlockstore) *BlockGetter { - return &BlockGetter{store: store} -} - -// BlockGetter is an adapter for dagstore.ReadBlockstore to implement blockservice.BlockGetter -// interface. -type BlockGetter struct { - store dagstore.ReadBlockstore -} - -// GetBlock gets the requested block by the given CID. -func (bg *BlockGetter) GetBlock(ctx context.Context, cid cid.Cid) (blocks.Block, error) { - return bg.store.Get(ctx, cid) -} - -// GetBlocks does a batch request for the given cids, returning blocks as -// they are found, in no particular order. -// -// It implements blockservice.BlockGetter interface, that requires: -// It may not be able to find all requested blocks (or the context may -// be canceled). In that case, it will close the channel early. It is up -// to the consumer to detect this situation and keep track which blocks -// it has received and which it hasn't. -func (bg *BlockGetter) GetBlocks(ctx context.Context, cids []cid.Cid) <-chan blocks.Block { - bCh := make(chan blocks.Block) - - go func() { - var wg sync.WaitGroup - wg.Add(len(cids)) - for _, c := range cids { - go func(cid cid.Cid) { - defer wg.Done() - block, err := bg.store.Get(ctx, cid) - if err != nil { - log.Debugw("getblocks: error getting block by cid", "cid", cid, "error", err) - return - } - - select { - case bCh <- block: - case <-ctx.Done(): - return - } - }(c) - } - wg.Wait() - close(bCh) - }() - - return bCh -} diff --git a/share/eds/adapters_test.go b/share/eds/adapters_test.go deleted file mode 100644 index 70165b81c8..0000000000 --- a/share/eds/adapters_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package eds - -import ( - "context" - "errors" - mrand "math/rand" - "sort" - "testing" - "time" - - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-node/share/ipld" -) - -func TestBlockGetter_GetBlocks(t *testing.T) { - t.Run("happy path", func(t *testing.T) { - cids := randCIDs(t, 32) - // sort cids in asc order - sort.Slice(cids, func(i, j int) bool { - return cids[i].String() < cids[j].String() - }) - - bg := &BlockGetter{store: rbsMock{}} - blocksCh := bg.GetBlocks(context.Background(), cids) - - // collect blocks from channel - blocks := make([]blocks.Block, 0, len(cids)) - for block := range blocksCh { - blocks = append(blocks, block) - } - - // sort blocks in cid asc order - sort.Slice(blocks, func(i, j int) bool { - return blocks[i].Cid().String() < blocks[j].Cid().String() - }) - - // validate results - require.Equal(t, len(cids), len(blocks)) - for i, block := range blocks { - require.Equal(t, cids[i].String(), block.Cid().String()) - } - }) - t.Run("retrieval error", func(t *testing.T) { - cids := randCIDs(t, 32) - - // split cids into failed and succeeded - failedLen := mrand.Intn(len(cids)-1) + 1 - failed := make(map[cid.Cid]struct{}, failedLen) - succeeded := make([]cid.Cid, 0, len(cids)-failedLen) - for i, cid := range cids { - if i < failedLen { - failed[cid] = struct{}{} - continue - } - succeeded = append(succeeded, cid) - } - - // sort succeeded cids in asc order - sort.Slice(succeeded, func(i, j int) bool { - return succeeded[i].String() < succeeded[j].String() - }) - - bg := &BlockGetter{store: rbsMock{failed: failed}} - blocksCh := bg.GetBlocks(context.Background(), cids) - - // collect blocks from channel - blocks := make([]blocks.Block, 0, len(cids)) - for block := range blocksCh { - blocks = append(blocks, block) - } - - // sort blocks in cid asc order - sort.Slice(blocks, func(i, j int) bool { - return blocks[i].Cid().String() < blocks[j].Cid().String() - }) - - // validate results - require.Equal(t, len(succeeded), len(blocks)) - for i, block := range blocks { - require.Equal(t, succeeded[i].String(), block.Cid().String()) - } - }) - t.Run("retrieval timeout", func(t *testing.T) { - cids := randCIDs(t, 128) - - bg := &BlockGetter{ - store: rbsMock{}, - } - - // cancel the context before any blocks are collected - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - blocksCh := bg.GetBlocks(ctx, cids) - - // pretend nobody is reading from blocksCh after context is canceled - time.Sleep(50 * time.Millisecond) - - // blocksCh should be closed indicating GetBlocks exited - select { - case _, ok := <-blocksCh: - require.False(t, ok) - default: - t.Error("channel is not closed on canceled context") - } - }) -} - -// rbsMock is a dagstore.ReadBlockstore mock -type rbsMock struct { - failed map[cid.Cid]struct{} -} - -func (r rbsMock) Has(context.Context, cid.Cid) (bool, error) { - panic("implement me") -} - -func (r rbsMock) Get(_ context.Context, cid cid.Cid) (blocks.Block, error) { - // return error for failed items - if _, ok := r.failed[cid]; ok { - return nil, errors.New("not found") - } - - return blocks.NewBlockWithCid(nil, cid) -} - -func (r rbsMock) GetSize(context.Context, cid.Cid) (int, error) { - panic("implement me") -} - -func (r rbsMock) AllKeysChan(context.Context) (<-chan cid.Cid, error) { - panic("implement me") -} - -func (r rbsMock) HashOnRead(bool) { - panic("implement me") -} - -func randCIDs(t *testing.T, n int) []cid.Cid { - cids := make([]cid.Cid, n) - for i := range cids { - cids[i] = ipld.RandNamespacedCID(t) - } - return cids -} diff --git a/share/eds/blockstore.go b/share/eds/blockstore.go deleted file mode 100644 index e44601870e..0000000000 --- a/share/eds/blockstore.go +++ /dev/null @@ -1,168 +0,0 @@ -package eds - -import ( - "context" - "errors" - "fmt" - - bstore "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/boxo/datastore/dshelp" - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" - ipld "github.com/ipfs/go-ipld-format" -) - -var _ bstore.Blockstore = (*blockstore)(nil) - -var ( - blockstoreCacheKey = datastore.NewKey("bs-cache") - errUnsupportedOperation = errors.New("unsupported operation") -) - -// blockstore implements the store.Blockstore interface on an EDSStore. -// The lru cache approach is heavily inspired by the existing implementation upstream. -// We simplified the design to not support multiple shards per key, call GetSize directly on the -// underlying RO blockstore, and do not throw errors on Put/PutMany. Also, we do not abstract away -// the blockstore operations. -// -// The intuition here is that each CAR file is its own blockstore, so we need this top level -// implementation to allow for the blockstore operations to be routed to the underlying stores. -type blockstore struct { - store *Store - ds datastore.Batching -} - -func newBlockstore(store *Store, ds datastore.Batching) *blockstore { - return &blockstore{ - store: store, - ds: namespace.Wrap(ds, blockstoreCacheKey), - } -} - -func (bs *blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) { - keys, err := bs.store.dgstr.ShardsContainingMultihash(ctx, cid.Hash()) - if errors.Is(err, ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { - // key wasn't found in top level blockstore, but could be in datastore while being reconstructed - dsHas, dsErr := bs.ds.Has(ctx, dshelp.MultihashToDsKey(cid.Hash())) - if dsErr != nil { - return false, nil - } - return dsHas, nil - } - if err != nil { - return false, err - } - - return len(keys) > 0, nil -} - -func (bs *blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) { - blockstr, err := bs.getReadOnlyBlockstore(ctx, cid) - if err == nil { - defer closeAndLog("blockstore", blockstr) - return blockstr.Get(ctx, cid) - } - - if errors.Is(err, ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { - k := dshelp.MultihashToDsKey(cid.Hash()) - blockData, err := bs.ds.Get(ctx, k) - if err == nil { - return blocks.NewBlockWithCid(blockData, cid) - } - // nmt's GetNode expects an ipld.ErrNotFound when a cid is not found. - return nil, ipld.ErrNotFound{Cid: cid} - } - - log.Debugf("failed to get blockstore for cid %s: %s", cid, err) - return nil, err -} - -func (bs *blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { - blockstr, err := bs.getReadOnlyBlockstore(ctx, cid) - if err == nil { - defer closeAndLog("blockstore", blockstr) - return blockstr.GetSize(ctx, cid) - } - - if errors.Is(err, ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { - k := dshelp.MultihashToDsKey(cid.Hash()) - size, err := bs.ds.GetSize(ctx, k) - if err == nil { - return size, nil - } - // nmt's GetSize expects an ipld.ErrNotFound when a cid is not found. - return 0, ipld.ErrNotFound{Cid: cid} - } - - log.Debugf("failed to get size for cid %s: %s", cid, err) - return 0, err -} - -func (bs *blockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error { - k := dshelp.MultihashToDsKey(cid.Hash()) - return bs.ds.Delete(ctx, k) -} - -func (bs *blockstore) Put(ctx context.Context, blk blocks.Block) error { - k := dshelp.MultihashToDsKey(blk.Cid().Hash()) - // note: we leave duplicate resolution to the underlying datastore - return bs.ds.Put(ctx, k, blk.RawData()) -} - -func (bs *blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { - if len(blocks) == 1 { - // performance fast-path - return bs.Put(ctx, blocks[0]) - } - - t, err := bs.ds.Batch(ctx) - if err != nil { - return err - } - for _, b := range blocks { - k := dshelp.MultihashToDsKey(b.Cid().Hash()) - err = t.Put(ctx, k, b.RawData()) - if err != nil { - return err - } - } - return t.Commit(ctx) -} - -// AllKeysChan is a noop on the EDS blockstore because the keys are not stored in a single CAR file. -func (bs *blockstore) AllKeysChan(context.Context) (<-chan cid.Cid, error) { - return nil, errUnsupportedOperation -} - -// HashOnRead is a noop on the EDS blockstore but an error cannot be returned due to the method -// signature from the blockstore interface. -func (bs *blockstore) HashOnRead(bool) { - log.Warnf("HashOnRead is a noop on the EDS blockstore") -} - -// getReadOnlyBlockstore finds the underlying blockstore of the shard that contains the given CID. -func (bs *blockstore) getReadOnlyBlockstore(ctx context.Context, cid cid.Cid) (*BlockstoreCloser, error) { - keys, err := bs.store.dgstr.ShardsContainingMultihash(ctx, cid.Hash()) - if errors.Is(err, datastore.ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { - return nil, ErrNotFound - } - if err != nil { - return nil, fmt.Errorf("failed to find shards containing multihash: %w", err) - } - - // check if either cache contains an accessor - shardKey := keys[0] - accessor, err := bs.store.cache.Load().Get(shardKey) - if err == nil { - return blockstoreCloser(accessor) - } - - // load accessor to the blockstore cache and use it as blockstoreCloser - accessor, err = bs.store.cache.Load().Second().GetOrLoad(ctx, shardKey, bs.store.getAccessor) - if err != nil { - return nil, fmt.Errorf("failed to get accessor for shard %s: %w", shardKey, err) - } - return blockstoreCloser(accessor) -} diff --git a/share/eds/blockstore_test.go b/share/eds/blockstore_test.go deleted file mode 100644 index d9dbf7ed30..0000000000 --- a/share/eds/blockstore_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package eds - -import ( - "context" - "io" - "testing" - - "github.com/filecoin-project/dagstore" - ipld "github.com/ipfs/go-ipld-format" - "github.com/ipld/go-car" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - ipld2 "github.com/celestiaorg/celestia-node/share/ipld" -) - -// TestBlockstore_Operations tests Has, Get, and GetSize on the top level eds.Store blockstore. -// It verifies that these operations are valid and successful on all blocks stored in a CAR file. -func TestBlockstore_Operations(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - edsStore, err := newStore(t) - require.NoError(t, err) - err = edsStore.Start(ctx) - require.NoError(t, err) - - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - - r, err := edsStore.GetCAR(ctx, dah.Hash()) - require.NoError(t, err) - carReader, err := car.NewCarReader(r) - require.NoError(t, err) - - topLevelBS := edsStore.Blockstore() - carBS, err := edsStore.CARBlockstore(ctx, dah.Hash()) - require.NoError(t, err) - defer func() { - require.NoError(t, carBS.Close()) - }() - - root, err := edsStore.GetDAH(ctx, dah.Hash()) - require.NoError(t, err) - require.True(t, dah.Equals(root)) - - blockstores := []dagstore.ReadBlockstore{topLevelBS, carBS} - - for { - next, err := carReader.Next() - if err != nil { - require.ErrorIs(t, err, io.EOF) - break - } - blockCid := next.Cid() - randomCid := ipld2.RandNamespacedCID(t) - - for _, bs := range blockstores { - // test GetSize - has, err := bs.Has(ctx, blockCid) - require.NoError(t, err, "blockstore.Has could not find root CID") - require.True(t, has) - - // test GetSize - block, err := bs.Get(ctx, blockCid) - assert.NoError(t, err, "blockstore.Get could not get a leaf CID") - assert.Equal(t, block.Cid(), blockCid) - assert.Equal(t, block.RawData(), next.RawData()) - - // test Get (cid not found) - _, err = bs.Get(ctx, randomCid) - require.ErrorAs(t, err, &ipld.ErrNotFound{Cid: randomCid}) - - // test GetSize - size, err := bs.GetSize(ctx, blockCid) - assert.NotZerof(t, size, "blocksize.GetSize reported a root block from blockstore was empty") - assert.NoError(t, err) - } - } -} diff --git a/share/eds/byzantine/bad_encoding.go b/share/eds/byzantine/bad_encoding.go index fbb6b592ea..b9b6b999d7 100644 --- a/share/eds/byzantine/bad_encoding.go +++ b/share/eds/byzantine/bad_encoding.go @@ -5,16 +5,21 @@ import ( "errors" "fmt" + logging "github.com/ipfs/go-log/v2" + "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/go-fraud" + "github.com/celestiaorg/nmt" + nmt_pb "github.com/celestiaorg/nmt/pb" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" pb "github.com/celestiaorg/celestia-node/share/eds/byzantine/pb" - "github.com/celestiaorg/celestia-node/share/ipld" ) +var log = logging.Logger("share/byzantine") + const ( version = "v0.1" @@ -27,7 +32,7 @@ type BadEncodingProof struct { // ShareWithProof contains all shares from row or col. // Shares that did not pass verification in rsmt2d will be nil. // For non-nil shares MerkleProofs are computed. - Shares []*ShareWithProof + Shares []*share.ShareWithProof // Index represents the row/col index where ErrByzantineRow/ErrByzantineColl occurred. Index uint32 // Axis represents the axis that verification failed on. @@ -70,7 +75,7 @@ func (p *BadEncodingProof) Height() uint64 { func (p *BadEncodingProof) MarshalBinary() ([]byte, error) { shares := make([]*pb.Share, 0, len(p.Shares)) for _, share := range p.Shares { - shares = append(shares, share.ShareWithProofToProto()) + shares = append(shares, ShareWithProofToProto(share)) } badEncodingFraudProof := pb.BadEncoding{ @@ -89,10 +94,11 @@ func (p *BadEncodingProof) UnmarshalBinary(data []byte) error { if err := in.Unmarshal(data); err != nil { return err } + axisType := rsmt2d.Axis(in.Axis) befp := &BadEncodingProof{ headerHash: in.HeaderHash, BlockHeight: in.Height, - Shares: ProtoToShare(in.Shares), + Shares: ProtoToShare(in.Shares, axisType), Index: in.Index, Axis: rsmt2d.Axis(in.Axis), } @@ -190,13 +196,11 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { continue } // validate inclusion of the share into one of the DAHeader roots - if ok := shr.Validate(ipld.MustCidFromNamespacedSha256(merkleRoots[index])); !ok { + if ok := shr.Validate(merkleRoots[index], index, int(p.Index), int(odsWidth)*2); !ok { log.Debugf("%s: %s at index %d", invalidProofPrefix, errIncorrectShare, index) return errIncorrectShare } - // NMTree commits the additional namespace while rsmt2d does not know about, so we trim it - // this is ugliness from NMTWrapper that we have to embrace ¯\_(ツ)_/¯ - shares[index] = share.GetData(shr.Share) + shares[index] = shr.Share } codec := share.DefaultRSMT2DCodec() @@ -208,7 +212,7 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { log.Debugw("failed to decode shares at height", "height", hdr.Height(), "err", err, ) - return nil + return fmt.Errorf("failed to decode shares: %w", err) } rebuiltExtendedShares, err := codec.Encode(rebuiltShares[0:odsWidth]) @@ -216,7 +220,7 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { log.Debugw("failed to encode shares at height", "height", hdr.Height(), "err", err, ) - return nil + return fmt.Errorf("failed to encode shares: %w", err) } copy(rebuiltShares[odsWidth:], rebuiltExtendedShares) @@ -227,7 +231,7 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { log.Debugw("failed to build a tree from the reconstructed shares at height", "height", hdr.Height(), "err", err, ) - return nil + return fmt.Errorf("failed to build a tree from the reconstructed shares: %w", err) } } @@ -236,7 +240,7 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { log.Debugw("failed to build a tree root at height", "height", hdr.Height(), "err", err, ) - return nil + return fmt.Errorf("failed to build a tree root: %w", err) } // root is a merkle root of the row/col where ErrByzantine occurred @@ -252,3 +256,45 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { } return nil } + +func ShareWithProofToProto(s *share.ShareWithProof) *pb.Share { + if s == nil { + return &pb.Share{} + } + + return &pb.Share{ + Data: s.Share, + Proof: &nmt_pb.Proof{ + Start: int64(s.Proof.Start()), + End: int64(s.Proof.End()), + Nodes: s.Proof.Nodes(), + LeafHash: s.Proof.LeafHash(), + IsMaxNamespaceIgnored: s.Proof.IsMaxNamespaceIDIgnored(), + }, + } +} + +func ProtoToShare(protoShares []*pb.Share, axisType rsmt2d.Axis) []*share.ShareWithProof { + shares := make([]*share.ShareWithProof, len(protoShares)) + for i, sh := range protoShares { + if sh.Proof == nil { + continue + } + proof := ProtoToProof(sh.Proof) + shares[i] = &share.ShareWithProof{ + Share: sh.Data, + Proof: &proof, + Axis: axisType, + } + } + return shares +} + +func ProtoToProof(protoProof *nmt_pb.Proof) nmt.Proof { + return nmt.NewInclusionProof( + int(protoProof.Start), + int(protoProof.End), + protoProof.Nodes, + protoProof.IsMaxNamespaceIgnored, + ) +} diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index e42e3c287c..cebf358280 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -21,9 +21,9 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/ipld" - "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" ) func TestBEFP_Validate(t *testing.T) { @@ -175,7 +175,7 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { rowShares := eds.Row(row) rowRoot := dah.RowRoots[row] - shareProofs, err := GetProofsForShares(ctx, bServ, ipld.MustCidFromNamespacedSha256(rowRoot), rowShares) + shareProofs, err := ipld.GetSharesWithProofs(ctx, bServ, rowRoot, rowShares, rsmt2d.Row) require.NoError(t, err) // create a fake error for data that was encoded correctly diff --git a/share/eds/byzantine/byzantine.go b/share/eds/byzantine/byzantine.go index d20b56deed..f78a2773bb 100644 --- a/share/eds/byzantine/byzantine.go +++ b/share/eds/byzantine/byzantine.go @@ -9,6 +9,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" ) @@ -19,7 +20,7 @@ import ( // Merkle Proof for each share. type ErrByzantine struct { Index uint32 - Shares []*ShareWithProof + Shares []*share.ShareWithProof Axis rsmt2d.Axis } @@ -40,11 +41,15 @@ func NewErrByzantine( dah.ColumnRoots, dah.RowRoots, }[errByz.Axis] + axisType := rsmt2d.Row + if errByz.Axis == rsmt2d.Row { + axisType = rsmt2d.Col + } - sharesWithProof := make([]*ShareWithProof, len(errByz.Shares)) + sharesWithProof := make([]*share.ShareWithProof, len(errByz.Shares)) type result struct { - share *ShareWithProof + share *share.ShareWithProof index int } resultCh := make(chan *result) @@ -55,16 +60,17 @@ func NewErrByzantine( index := index go func() { - share, err := getProofsAt( + sh, err := ipld.GetShareWithProof( ctx, bGetter, - ipld.MustCidFromNamespacedSha256(roots[index]), + roots[index], int(errByz.Index), len(errByz.Shares), + axisType, ) if err != nil { log.Warn("requesting proof failed", "root", roots[index], "err", err) return } - resultCh <- &result{share, index} + resultCh <- &result{sh, index} }() } diff --git a/share/eds/byzantine/share_proof.go b/share/eds/byzantine/share_proof.go deleted file mode 100644 index 98b58ebbec..0000000000 --- a/share/eds/byzantine/share_proof.go +++ /dev/null @@ -1,134 +0,0 @@ -package byzantine - -import ( - "context" - "crypto/sha256" - - "github.com/ipfs/boxo/blockservice" - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - - "github.com/celestiaorg/nmt" - nmt_pb "github.com/celestiaorg/nmt/pb" - - "github.com/celestiaorg/celestia-node/share" - pb "github.com/celestiaorg/celestia-node/share/eds/byzantine/pb" - "github.com/celestiaorg/celestia-node/share/ipld" -) - -var log = logging.Logger("share/byzantine") - -// ShareWithProof contains data with corresponding Merkle Proof -type ShareWithProof struct { - // Share is a full data including namespace - share.Share - // Proof is a Merkle Proof of current share - Proof *nmt.Proof -} - -// NewShareWithProof takes the given leaf and its path, starting from the tree root, -// and computes the nmt.Proof for it. -func NewShareWithProof(index int, share share.Share, pathToLeaf []cid.Cid) *ShareWithProof { - rangeProofs := make([][]byte, 0, len(pathToLeaf)) - for i := len(pathToLeaf) - 1; i >= 0; i-- { - node := ipld.NamespacedSha256FromCID(pathToLeaf[i]) - rangeProofs = append(rangeProofs, node) - } - - proof := nmt.NewInclusionProof(index, index+1, rangeProofs, true) - return &ShareWithProof{ - share, - &proof, - } -} - -// Validate validates inclusion of the share under the given root CID. -func (s *ShareWithProof) Validate(root cid.Cid) bool { - return s.Proof.VerifyInclusion( - sha256.New(), // TODO(@Wondertan): This should be defined somewhere globally - share.GetNamespace(s.Share).ToNMT(), - [][]byte{share.GetData(s.Share)}, - ipld.NamespacedSha256FromCID(root), - ) -} - -func (s *ShareWithProof) ShareWithProofToProto() *pb.Share { - if s == nil { - return &pb.Share{} - } - - return &pb.Share{ - Data: s.Share, - Proof: &nmt_pb.Proof{ - Start: int64(s.Proof.Start()), - End: int64(s.Proof.End()), - Nodes: s.Proof.Nodes(), - LeafHash: s.Proof.LeafHash(), - IsMaxNamespaceIgnored: s.Proof.IsMaxNamespaceIDIgnored(), - }, - } -} - -// GetProofsForShares fetches Merkle proofs for the given shares -// and returns the result as an array of ShareWithProof. -func GetProofsForShares( - ctx context.Context, - bGetter blockservice.BlockGetter, - root cid.Cid, - shares [][]byte, -) ([]*ShareWithProof, error) { - proofs := make([]*ShareWithProof, len(shares)) - for index, share := range shares { - if share != nil { - proof, err := getProofsAt(ctx, bGetter, root, index, len(shares)) - if err != nil { - return nil, err - } - proofs[index] = proof - } - } - return proofs, nil -} - -func getProofsAt( - ctx context.Context, - bGetter blockservice.BlockGetter, - root cid.Cid, - index, - total int, -) (*ShareWithProof, error) { - proof := make([]cid.Cid, 0) - // TODO(@vgonkivs): Combine GetLeafData and GetProof in one function as the are traversing the same - // tree. Add options that will control what data will be fetched. - node, err := ipld.GetLeaf(ctx, bGetter, root, index, total) - if err != nil { - return nil, err - } - - proof, err = ipld.GetProof(ctx, bGetter, root, proof, index, total) - if err != nil { - return nil, err - } - return NewShareWithProof(index, node.RawData(), proof), nil -} - -func ProtoToShare(protoShares []*pb.Share) []*ShareWithProof { - shares := make([]*ShareWithProof, len(protoShares)) - for i, share := range protoShares { - if share.Proof == nil { - continue - } - proof := ProtoToProof(share.Proof) - shares[i] = &ShareWithProof{share.Data, &proof} - } - return shares -} - -func ProtoToProof(protoProof *nmt_pb.Proof) nmt.Proof { - return nmt.NewInclusionProof( - int(protoProof.Start), - int(protoProof.End), - protoProof.Nodes, - protoProof.IsMaxNamespaceIgnored, - ) -} diff --git a/share/eds/byzantine/share_proof_test.go b/share/eds/byzantine/share_proof_test.go deleted file mode 100644 index a9021d806d..0000000000 --- a/share/eds/byzantine/share_proof_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package byzantine - -import ( - "context" - "strconv" - "testing" - "time" - - "github.com/ipfs/go-cid" - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-app/pkg/da" - - "github.com/celestiaorg/celestia-node/share/ipld" - "github.com/celestiaorg/celestia-node/share/sharetest" -) - -func TestGetProof(t *testing.T) { - const width = 4 - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) - defer cancel() - bServ := ipld.NewMemBlockservice() - - shares := sharetest.RandShares(t, width*width) - in, err := ipld.AddShares(ctx, shares, bServ) - require.NoError(t, err) - - dah, err := da.NewDataAvailabilityHeader(in) - require.NoError(t, err) - var tests = []struct { - roots [][]byte - }{ - {dah.RowRoots}, - {dah.ColumnRoots}, - } - - for i, tt := range tests { - t.Run(strconv.Itoa(i), func(t *testing.T) { - for _, root := range tt.roots { - rootCid := ipld.MustCidFromNamespacedSha256(root) - for index := 0; uint(index) < in.Width(); index++ { - proof := make([]cid.Cid, 0) - proof, err = ipld.GetProof(ctx, bServ, rootCid, proof, index, int(in.Width())) - require.NoError(t, err) - node, err := ipld.GetLeaf(ctx, bServ, rootCid, index, int(in.Width())) - require.NoError(t, err) - inclusion := NewShareWithProof(index, node.RawData(), proof) - require.True(t, inclusion.Validate(rootCid)) - } - } - }) - } -} - -func TestGetProofs(t *testing.T) { - const width = 4 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) - defer cancel() - bServ := ipld.NewMemBlockservice() - - shares := sharetest.RandShares(t, width*width) - in, err := ipld.AddShares(ctx, shares, bServ) - require.NoError(t, err) - - dah, err := da.NewDataAvailabilityHeader(in) - require.NoError(t, err) - for _, root := range dah.ColumnRoots { - rootCid := ipld.MustCidFromNamespacedSha256(root) - data := make([][]byte, 0, in.Width()) - for index := 0; uint(index) < in.Width(); index++ { - node, err := ipld.GetLeaf(ctx, bServ, rootCid, index, int(in.Width())) - require.NoError(t, err) - data = append(data, node.RawData()[9:]) - } - - proves, err := GetProofsForShares(ctx, bServ, rootCid, data) - require.NoError(t, err) - for _, proof := range proves { - require.True(t, proof.Validate(rootCid)) - } - } -} diff --git a/share/eds/cache/accessor_cache.go b/share/eds/cache/accessor_cache.go deleted file mode 100644 index 6f937818f8..0000000000 --- a/share/eds/cache/accessor_cache.go +++ /dev/null @@ -1,262 +0,0 @@ -package cache - -import ( - "context" - "errors" - "fmt" - "io" - "sync" - "sync/atomic" - "time" - - "github.com/filecoin-project/dagstore" - "github.com/filecoin-project/dagstore/shard" - lru "github.com/hashicorp/golang-lru/v2" -) - -const defaultCloseTimeout = time.Minute - -var _ Cache = (*AccessorCache)(nil) - -// AccessorCache implements the Cache interface using an LRU cache backend. -type AccessorCache struct { - // The name is a prefix that will be used for cache metrics if they are enabled. - name string - // stripedLocks prevents simultaneous RW access to the blockstore cache for a shard. Instead - // of using only one lock or one lock per key, we stripe the shard keys across 256 locks. 256 is - // chosen because it 0-255 is the range of values we get looking at the last byte of the key. - stripedLocks [256]sync.Mutex - // Caches the blockstore for a given shard for shard read affinity, i.e., further reads will likely - // be from the same shard. Maps (shard key -> blockstore). - cache *lru.Cache[shard.Key, *accessorWithBlockstore] - - metrics *metrics -} - -// accessorWithBlockstore is the value that we store in the blockstore Cache. It implements the -// Accessor interface. -type accessorWithBlockstore struct { - sync.RWMutex - shardAccessor Accessor - // The blockstore is stored separately because each access to the blockstore over the shard - // accessor reopens the underlying CAR. - bs dagstore.ReadBlockstore - - done chan struct{} - refs atomic.Int32 - isClosed bool -} - -// Blockstore implements the Blockstore of the Accessor interface. It creates the blockstore on the -// first request and reuses the created instance for all subsequent requests. -func (s *accessorWithBlockstore) Blockstore() (dagstore.ReadBlockstore, error) { - s.Lock() - defer s.Unlock() - var err error - if s.bs == nil { - s.bs, err = s.shardAccessor.Blockstore() - } - return s.bs, err -} - -// Reader returns a new copy of the reader to read data. -func (s *accessorWithBlockstore) Reader() io.Reader { - return s.shardAccessor.Reader() -} - -func (s *accessorWithBlockstore) addRef() error { - s.Lock() - defer s.Unlock() - if s.isClosed { - // item is already closed and soon will be removed after all refs are released - return errCacheMiss - } - if s.refs.Add(1) == 1 { - // there were no refs previously and done channel was closed, reopen it by recreating - s.done = make(chan struct{}) - } - return nil -} - -func (s *accessorWithBlockstore) removeRef() { - s.Lock() - defer s.Unlock() - if s.refs.Add(-1) <= 0 { - close(s.done) - } -} - -func (s *accessorWithBlockstore) close() error { - s.Lock() - if s.isClosed { - s.Unlock() - // accessor will be closed by another goroutine - return nil - } - s.isClosed = true - done := s.done - s.Unlock() - - select { - case <-done: - case <-time.After(defaultCloseTimeout): - return fmt.Errorf("closing accessor, some readers didn't close the accessor within timeout,"+ - " amount left: %v", s.refs.Load()) - } - if err := s.shardAccessor.Close(); err != nil { - return fmt.Errorf("closing accessor: %w", err) - } - return nil -} - -func NewAccessorCache(name string, cacheSize int) (*AccessorCache, error) { - bc := &AccessorCache{ - name: name, - } - // Instantiate the blockstore Cache. - bslru, err := lru.NewWithEvict[shard.Key, *accessorWithBlockstore](cacheSize, bc.evictFn()) - if err != nil { - return nil, fmt.Errorf("failed to instantiate blockstore cache: %w", err) - } - bc.cache = bslru - return bc, nil -} - -// evictFn will be invoked when an item is evicted from the cache. -func (bc *AccessorCache) evictFn() func(shard.Key, *accessorWithBlockstore) { - return func(_ shard.Key, abs *accessorWithBlockstore) { - // we can release accessor from cache early, while it is being closed in parallel routine - go func() { - err := abs.close() - if err != nil { - bc.metrics.observeEvicted(true) - log.Errorf("couldn't close accessor after cache eviction: %s", err) - return - } - bc.metrics.observeEvicted(false) - }() - } -} - -// Get retrieves the Accessor for a given shard key from the Cache. If the Accessor is not in -// the Cache, it returns an errCacheMiss. -func (bc *AccessorCache) Get(key shard.Key) (Accessor, error) { - lk := &bc.stripedLocks[shardKeyToStriped(key)] - lk.Lock() - defer lk.Unlock() - - accessor, err := bc.get(key) - if err != nil { - bc.metrics.observeGet(false) - return nil, err - } - bc.metrics.observeGet(true) - return newRefCloser(accessor) -} - -func (bc *AccessorCache) get(key shard.Key) (*accessorWithBlockstore, error) { - abs, ok := bc.cache.Get(key) - if !ok { - return nil, errCacheMiss - } - return abs, nil -} - -// GetOrLoad attempts to get an item from the cache, and if not found, invokes -// the provided loader function to load it. -func (bc *AccessorCache) GetOrLoad( - ctx context.Context, - key shard.Key, - loader func(context.Context, shard.Key) (Accessor, error), -) (Accessor, error) { - lk := &bc.stripedLocks[shardKeyToStriped(key)] - lk.Lock() - defer lk.Unlock() - - abs, err := bc.get(key) - if err == nil { - // return accessor, only of it is not closed yet - accessorWithRef, err := newRefCloser(abs) - if err == nil { - bc.metrics.observeGet(true) - return accessorWithRef, nil - } - } - - // accessor not found in cache, so load new one using loader - accessor, err := loader(ctx, key) - if err != nil { - return nil, fmt.Errorf("unable to load accessor: %w", err) - } - - abs = &accessorWithBlockstore{ - shardAccessor: accessor, - } - - // Create a new accessor first to increment the reference count in it, so it cannot get evicted - // from the inner lru cache before it is used. - accessorWithRef, err := newRefCloser(abs) - if err != nil { - return nil, err - } - bc.cache.Add(key, abs) - return accessorWithRef, nil -} - -// Remove removes the Accessor for a given key from the cache. -func (bc *AccessorCache) Remove(key shard.Key) error { - lk := &bc.stripedLocks[shardKeyToStriped(key)] - lk.Lock() - accessor, err := bc.get(key) - lk.Unlock() - if errors.Is(err, errCacheMiss) { - // item is not in cache - return nil - } - if err = accessor.close(); err != nil { - return err - } - // The cache will call evictFn on removal, where accessor close will be called. - bc.cache.Remove(key) - return nil -} - -// EnableMetrics enables metrics for the cache. -func (bc *AccessorCache) EnableMetrics() error { - var err error - bc.metrics, err = newMetrics(bc) - return err -} - -// refCloser manages references to accessor from provided reader and removes the ref, when the -// Close is called -type refCloser struct { - *accessorWithBlockstore - closeFn func() -} - -// newRefCloser creates new refCloser -func newRefCloser(abs *accessorWithBlockstore) (*refCloser, error) { - if err := abs.addRef(); err != nil { - return nil, err - } - - var closeOnce sync.Once - return &refCloser{ - accessorWithBlockstore: abs, - closeFn: func() { - closeOnce.Do(abs.removeRef) - }, - }, nil -} - -func (c *refCloser) Close() error { - c.closeFn() - return nil -} - -// shardKeyToStriped returns the index of the lock to use for a given shard key. We use the last -// byte of the shard key as the pseudo-random index. -func shardKeyToStriped(sk shard.Key) byte { - return sk.String()[len(sk.String())-1] -} diff --git a/share/eds/cache/cache.go b/share/eds/cache/cache.go deleted file mode 100644 index 13e207d7c0..0000000000 --- a/share/eds/cache/cache.go +++ /dev/null @@ -1,49 +0,0 @@ -package cache - -import ( - "context" - "errors" - "io" - - "github.com/filecoin-project/dagstore" - "github.com/filecoin-project/dagstore/shard" - logging "github.com/ipfs/go-log/v2" - "go.opentelemetry.io/otel" -) - -var ( - log = logging.Logger("share/eds/cache") - meter = otel.Meter("eds_store_cache") -) - -var ( - errCacheMiss = errors.New("accessor not found in blockstore cache") -) - -// Cache is an interface that defines the basic Cache operations. -type Cache interface { - // Get retrieves an item from the Cache. - Get(shard.Key) (Accessor, error) - - // GetOrLoad attempts to get an item from the Cache and, if not found, invokes - // the provided loader function to load it into the Cache. - GetOrLoad( - ctx context.Context, - key shard.Key, - loader func(context.Context, shard.Key) (Accessor, error), - ) (Accessor, error) - - // Remove removes an item from Cache. - Remove(shard.Key) error - - // EnableMetrics enables metrics in Cache - EnableMetrics() error -} - -// Accessor is a interface type returned by cache, that allows to read raw data by reader or create -// readblockstore -type Accessor interface { - Blockstore() (dagstore.ReadBlockstore, error) - Reader() io.Reader - io.Closer -} diff --git a/share/eds/cache/noop.go b/share/eds/cache/noop.go deleted file mode 100644 index 0a1a39ec7e..0000000000 --- a/share/eds/cache/noop.go +++ /dev/null @@ -1,50 +0,0 @@ -package cache - -import ( - "context" - "io" - - "github.com/filecoin-project/dagstore" - "github.com/filecoin-project/dagstore/shard" -) - -var _ Cache = (*NoopCache)(nil) - -// NoopCache implements noop version of Cache interface -type NoopCache struct{} - -func (n NoopCache) Get(shard.Key) (Accessor, error) { - return nil, errCacheMiss -} - -func (n NoopCache) GetOrLoad( - context.Context, shard.Key, - func(context.Context, shard.Key) (Accessor, error), -) (Accessor, error) { - return NoopAccessor{}, nil -} - -func (n NoopCache) Remove(shard.Key) error { - return nil -} - -func (n NoopCache) EnableMetrics() error { - return nil -} - -var _ Accessor = (*NoopAccessor)(nil) - -// NoopAccessor implements noop version of Accessor interface -type NoopAccessor struct{} - -func (n NoopAccessor) Blockstore() (dagstore.ReadBlockstore, error) { - return nil, nil -} - -func (n NoopAccessor) Reader() io.Reader { - return nil -} - -func (n NoopAccessor) Close() error { - return nil -} diff --git a/share/eds/eds.go b/share/eds/eds.go deleted file mode 100644 index e0433a1b6b..0000000000 --- a/share/eds/eds.go +++ /dev/null @@ -1,274 +0,0 @@ -package eds - -import ( - "bytes" - "context" - "crypto/sha256" - "errors" - "fmt" - "io" - "math" - - "github.com/ipfs/go-cid" - "github.com/ipld/go-car" - "github.com/ipld/go-car/util" - - "github.com/celestiaorg/celestia-app/pkg/wrapper" - "github.com/celestiaorg/nmt" - "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/libs/utils" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/ipld" -) - -var ErrEmptySquare = errors.New("share: importing empty data") - -// WriteEDS writes the entire EDS into the given io.Writer as CARv1 file. -// This includes all shares in quadrant order, followed by all inner nodes of the NMT tree. -// Order: [ Carv1Header | Q1 | Q2 | Q3 | Q4 | inner nodes ] -// For more information about the header: https://ipld.io/specs/transport/car/carv1/#header -func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) (err error) { - ctx, span := tracer.Start(ctx, "write-eds") - defer func() { - utils.SetStatusAndEnd(span, err) - }() - - // Creates and writes Carv1Header. Roots are the eds Row + Col roots - err = writeHeader(eds, w) - if err != nil { - return fmt.Errorf("share: writing carv1 header: %w", err) - } - // Iterates over shares in quadrant order via eds.GetCell - err = writeQuadrants(eds, w) - if err != nil { - return fmt.Errorf("share: writing shares: %w", err) - } - - // Iterates over proofs and writes them to the CAR - err = writeProofs(ctx, eds, w) - if err != nil { - return fmt.Errorf("share: writing proofs: %w", err) - } - return nil -} - -// writeHeader creates a CarV1 header using the EDS's Row and Column roots as the list of DAG roots. -func writeHeader(eds *rsmt2d.ExtendedDataSquare, w io.Writer) error { - rootCids, err := rootsToCids(eds) - if err != nil { - return fmt.Errorf("getting root cids: %w", err) - } - - return car.WriteHeader(&car.CarHeader{ - Roots: rootCids, - Version: 1, - }, w) -} - -// writeQuadrants reorders the shares to quadrant order and writes them to the CARv1 file. -func writeQuadrants(eds *rsmt2d.ExtendedDataSquare, w io.Writer) error { - hasher := nmt.NewNmtHasher(sha256.New(), share.NamespaceSize, ipld.NMTIgnoreMaxNamespace) - shares := quadrantOrder(eds) - for _, share := range shares { - leaf, err := hasher.HashLeaf(share) - if err != nil { - return fmt.Errorf("hashing share: %w", err) - } - cid, err := ipld.CidFromNamespacedSha256(leaf) - if err != nil { - return fmt.Errorf("getting cid from share: %w", err) - } - err = util.LdWrite(w, cid.Bytes(), share) - if err != nil { - return fmt.Errorf("writing share to file: %w", err) - } - } - return nil -} - -// writeProofs iterates over the in-memory blockstore's keys and writes all inner nodes to the -// CARv1 file. -func writeProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) error { - // check if proofs are collected by ipld.ProofsAdder in previous reconstructions of eds - proofs, err := getProofs(ctx, eds) - if err != nil { - return fmt.Errorf("recomputing proofs: %w", err) - } - - for id, proof := range proofs { - err := util.LdWrite(w, id.Bytes(), proof) - if err != nil { - return fmt.Errorf("writing proof to the car: %w", err) - } - } - return nil -} - -func getProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare) (map[cid.Cid][]byte, error) { - // check if there are proofs collected by ipld.ProofsAdder in previous reconstruction of eds - if adder := ipld.ProofsAdderFromCtx(ctx); adder != nil { - defer adder.Purge() - return adder.Proofs(), nil - } - - // recompute proofs from eds - shares := eds.Flattened() - shareCount := len(shares) - if shareCount == 0 { - return nil, ErrEmptySquare - } - odsWidth := int(math.Sqrt(float64(shareCount)) / 2) - - // this adder ignores leaves, so that they are not added to the store we iterate through in - // writeProofs - adder := ipld.NewProofsAdder(odsWidth * 2) - defer adder.Purge() - - eds, err := rsmt2d.ImportExtendedDataSquare( - shares, - share.DefaultRSMT2DCodec(), - wrapper.NewConstructor(uint64(odsWidth), - nmt.NodeVisitor(adder.VisitFn())), - ) - if err != nil { - return nil, fmt.Errorf("recomputing data square: %w", err) - } - // compute roots - if _, err = eds.RowRoots(); err != nil { - return nil, fmt.Errorf("computing row roots: %w", err) - } - - return adder.Proofs(), nil -} - -// quadrantOrder reorders the shares in the EDS to quadrant row-by-row order, prepending the -// respective namespace to the shares. -// e.g. [ Q1 R1 | Q1 R2 | Q1 R3 | Q1 R4 | Q2 R1 | Q2 R2 .... ] -func quadrantOrder(eds *rsmt2d.ExtendedDataSquare) [][]byte { - size := eds.Width() * eds.Width() - shares := make([][]byte, size) - - quadrantWidth := int(eds.Width() / 2) - quadrantSize := quadrantWidth * quadrantWidth - for i := 0; i < quadrantWidth; i++ { - for j := 0; j < quadrantWidth; j++ { - cells := getQuadrantCells(eds, uint(i), uint(j)) - innerOffset := i*quadrantWidth + j - for quadrant := 0; quadrant < 4; quadrant++ { - shares[(quadrant*quadrantSize)+innerOffset] = prependNamespace(quadrant, cells[quadrant]) - } - } - } - return shares -} - -// getQuadrantCells returns the cell of each EDS quadrant with the passed inner-quadrant coordinates -func getQuadrantCells(eds *rsmt2d.ExtendedDataSquare, i, j uint) [][]byte { - cells := make([][]byte, 4) - quadrantWidth := eds.Width() / 2 - cells[0] = eds.GetCell(i, j) - cells[1] = eds.GetCell(i, j+quadrantWidth) - cells[2] = eds.GetCell(i+quadrantWidth, j) - cells[3] = eds.GetCell(i+quadrantWidth, j+quadrantWidth) - return cells -} - -// prependNamespace adds the namespace to the passed share if in the first quadrant, -// otherwise it adds the ParitySharesNamespace to the beginning. -func prependNamespace(quadrant int, shr share.Share) []byte { - namespacedShare := make([]byte, 0, share.NamespaceSize+share.Size) - switch quadrant { - case 0: - return append(append(namespacedShare, share.GetNamespace(shr)...), shr...) - case 1, 2, 3: - return append(append(namespacedShare, share.ParitySharesNamespace...), shr...) - default: - panic("invalid quadrant") - } -} - -// rootsToCids converts the EDS's Row and Column roots to CIDs. -func rootsToCids(eds *rsmt2d.ExtendedDataSquare) ([]cid.Cid, error) { - rowRoots, err := eds.RowRoots() - if err != nil { - return nil, err - } - colRoots, err := eds.ColRoots() - if err != nil { - return nil, err - } - - roots := make([][]byte, 0, len(rowRoots)+len(colRoots)) - roots = append(roots, rowRoots...) - roots = append(roots, colRoots...) - rootCids := make([]cid.Cid, len(roots)) - for i, r := range roots { - rootCids[i], err = ipld.CidFromNamespacedSha256(r) - if err != nil { - return nil, fmt.Errorf("getting cid from root: %w", err) - } - } - return rootCids, nil -} - -// ReadEDS reads the first EDS quadrant (1/4) from an io.Reader CAR file. -// Only the first quadrant will be read, which represents the original data. -// The returned EDS is guaranteed to be full and valid against the DataRoot, otherwise ReadEDS -// errors. -func ReadEDS(ctx context.Context, r io.Reader, root share.DataHash) (eds *rsmt2d.ExtendedDataSquare, err error) { - _, span := tracer.Start(ctx, "read-eds") - defer func() { - utils.SetStatusAndEnd(span, err) - }() - - carReader, err := car.NewCarReader(r) - if err != nil { - return nil, fmt.Errorf("share: reading car file: %w", err) - } - - // car header includes both row and col roots in header - odsWidth := len(carReader.Header.Roots) / 4 - odsSquareSize := odsWidth * odsWidth - shares := make([][]byte, odsSquareSize) - // the first quadrant is stored directly after the header, - // so we can just read the first odsSquareSize blocks - for i := 0; i < odsSquareSize; i++ { - block, err := carReader.Next() - if err != nil { - return nil, fmt.Errorf("share: reading next car entry: %w", err) - } - // the stored first quadrant shares are wrapped with the namespace twice. - // we cut it off here, because it is added again while importing to the tree below - shares[i] = share.GetData(block.RawData()) - } - - // use proofs adder if provided, to cache collected proofs while recomputing the eds - var opts []nmt.Option - visitor := ipld.ProofsAdderFromCtx(ctx).VisitFn() - if visitor != nil { - opts = append(opts, nmt.NodeVisitor(visitor)) - } - - eds, err = rsmt2d.ComputeExtendedDataSquare( - shares, - share.DefaultRSMT2DCodec(), - wrapper.NewConstructor(uint64(odsWidth), opts...), - ) - if err != nil { - return nil, fmt.Errorf("share: computing eds: %w", err) - } - - newDah, err := share.NewRoot(eds) - if err != nil { - return nil, err - } - if !bytes.Equal(newDah.Hash(), root) { - return nil, fmt.Errorf( - "share: content integrity mismatch: imported root %s doesn't match expected root %s", - newDah.Hash(), - root, - ) - } - return eds, nil -} diff --git a/share/eds/eds_test.go b/share/eds/eds_test.go deleted file mode 100644 index ffb05343b9..0000000000 --- a/share/eds/eds_test.go +++ /dev/null @@ -1,283 +0,0 @@ -package eds - -import ( - "bytes" - "context" - "embed" - "encoding/json" - "fmt" - "os" - "testing" - - bstore "github.com/ipfs/boxo/blockstore" - ds "github.com/ipfs/go-datastore" - carv1 "github.com/ipld/go-car" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/libs/rand" - - "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds/edstest" -) - -//go:embed "testdata/example-root.json" -var exampleRoot string - -//go:embed "testdata/example.car" -var f embed.FS - -func TestQuadrantOrder(t *testing.T) { - testCases := []struct { - name string - squareSize int - }{ - {"smol", 2}, - {"still smol", 8}, - {"default mainnet", appconsts.DefaultGovMaxSquareSize}, - {"max", share.MaxSquareSize}, - } - - testShareSize := 64 - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - shares := make([][]byte, tc.squareSize*tc.squareSize) - - for i := 0; i < tc.squareSize*tc.squareSize; i++ { - shares[i] = rand.Bytes(testShareSize) - } - - eds, err := rsmt2d.ComputeExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), rsmt2d.NewDefaultTree) - require.NoError(t, err) - - res := quadrantOrder(eds) - for _, s := range res { - require.Len(t, s, testShareSize+share.NamespaceSize) - } - - for q := 0; q < 4; q++ { - for i := 0; i < tc.squareSize; i++ { - for j := 0; j < tc.squareSize; j++ { - resIndex := q*tc.squareSize*tc.squareSize + i*tc.squareSize + j - edsRow := q/2*tc.squareSize + i - edsCol := (q%2)*tc.squareSize + j - - assert.Equal(t, res[resIndex], prependNamespace(q, eds.Row(uint(edsRow))[edsCol])) - } - } - } - }) - } -} - -func TestWriteEDS(t *testing.T) { - writeRandomEDS(t) -} - -func TestWriteEDSHeaderRoots(t *testing.T) { - eds := writeRandomEDS(t) - f := openWrittenEDS(t) - defer f.Close() - - reader, err := carv1.NewCarReader(f) - require.NoError(t, err, "error creating car reader") - roots, err := rootsToCids(eds) - require.NoError(t, err, "error converting roots to cids") - require.Equal(t, roots, reader.Header.Roots) -} - -func TestWriteEDSStartsWithLeaves(t *testing.T) { - eds := writeRandomEDS(t) - f := openWrittenEDS(t) - defer f.Close() - - reader, err := carv1.NewCarReader(f) - require.NoError(t, err, "error creating car reader") - block, err := reader.Next() - require.NoError(t, err, "error getting first block") - - require.Equal(t, share.GetData(block.RawData()), eds.GetCell(0, 0)) -} - -func TestWriteEDSIncludesRoots(t *testing.T) { - writeRandomEDS(t) - f := openWrittenEDS(t) - defer f.Close() - - bs := bstore.NewBlockstore(ds.NewMapDatastore()) - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - loaded, err := carv1.LoadCar(ctx, bs, f) - require.NoError(t, err, "error loading car file") - for _, root := range loaded.Roots { - ok, err := bs.Has(context.Background(), root) - require.NoError(t, err, "error checking if blockstore has root") - require.True(t, ok, "blockstore does not have root") - } -} - -func TestWriteEDSInQuadrantOrder(t *testing.T) { - eds := writeRandomEDS(t) - f := openWrittenEDS(t) - defer f.Close() - - reader, err := carv1.NewCarReader(f) - require.NoError(t, err, "error creating car reader") - - shares := quadrantOrder(eds) - for i := 0; i < len(shares); i++ { - block, err := reader.Next() - require.NoError(t, err, "error getting block") - require.Equal(t, block.RawData(), shares[i]) - } -} - -func TestReadWriteRoundtrip(t *testing.T) { - eds := writeRandomEDS(t) - dah, err := share.NewRoot(eds) - require.NoError(t, err) - f := openWrittenEDS(t) - defer f.Close() - - loaded, err := ReadEDS(context.Background(), f, dah.Hash()) - require.NoError(t, err, "error reading EDS from file") - - rowRoots, err := eds.RowRoots() - require.NoError(t, err) - loadedRowRoots, err := loaded.RowRoots() - require.NoError(t, err) - require.Equal(t, rowRoots, loadedRowRoots) - - colRoots, err := eds.ColRoots() - require.NoError(t, err) - loadedColRoots, err := loaded.ColRoots() - require.NoError(t, err) - require.Equal(t, colRoots, loadedColRoots) -} - -func TestReadEDS(t *testing.T) { - f, err := f.Open("testdata/example.car") - require.NoError(t, err, "error opening file") - - var dah da.DataAvailabilityHeader - err = json.Unmarshal([]byte(exampleRoot), &dah) - require.NoError(t, err, "error unmarshaling example root") - - loaded, err := ReadEDS(context.Background(), f, dah.Hash()) - require.NoError(t, err, "error reading EDS from file") - rowRoots, err := loaded.RowRoots() - require.NoError(t, err) - require.Equal(t, dah.RowRoots, rowRoots) - colRoots, err := loaded.ColRoots() - require.NoError(t, err) - require.Equal(t, dah.ColumnRoots, colRoots) -} - -func TestReadEDSContentIntegrityMismatch(t *testing.T) { - writeRandomEDS(t) - dah, err := da.NewDataAvailabilityHeader(edstest.RandEDS(t, 4)) - require.NoError(t, err) - f := openWrittenEDS(t) - defer f.Close() - - _, err = ReadEDS(context.Background(), f, dah.Hash()) - require.ErrorContains(t, err, "share: content integrity mismatch: imported root") -} - -// BenchmarkReadWriteEDS benchmarks the time it takes to write and read an EDS from disk. The -// benchmark is run with a 4x4 ODS to a 64x64 ODS - a higher value can be used, but it will run for -// much longer. -func BenchmarkReadWriteEDS(b *testing.B) { - ctx, cancel := context.WithCancel(context.Background()) - b.Cleanup(cancel) - for originalDataWidth := 4; originalDataWidth <= 64; originalDataWidth *= 2 { - eds := edstest.RandEDS(b, originalDataWidth) - dah, err := share.NewRoot(eds) - require.NoError(b, err) - b.Run(fmt.Sprintf("Writing %dx%d", originalDataWidth, originalDataWidth), func(b *testing.B) { - b.ReportAllocs() - for i := 0; i < b.N; i++ { - f := new(bytes.Buffer) - err := WriteEDS(ctx, eds, f) - require.NoError(b, err) - } - }) - b.Run(fmt.Sprintf("Reading %dx%d", originalDataWidth, originalDataWidth), func(b *testing.B) { - b.ReportAllocs() - for i := 0; i < b.N; i++ { - b.StopTimer() - f := new(bytes.Buffer) - _ = WriteEDS(ctx, eds, f) - b.StartTimer() - _, err := ReadEDS(ctx, f, dah.Hash()) - require.NoError(b, err) - } - }) - } -} - -func writeRandomEDS(t *testing.T) *rsmt2d.ExtendedDataSquare { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - tmpDir := t.TempDir() - err := os.Chdir(tmpDir) - require.NoError(t, err, "error changing to the temporary test directory") - f, err := os.OpenFile("test.car", os.O_WRONLY|os.O_CREATE, 0600) - require.NoError(t, err, "error opening file") - - eds := edstest.RandEDS(t, 4) - err = WriteEDS(ctx, eds, f) - require.NoError(t, err, "error writing EDS to file") - f.Close() - return eds -} - -func openWrittenEDS(t *testing.T) *os.File { - t.Helper() - f, err := os.OpenFile("test.car", os.O_RDONLY, 0600) - require.NoError(t, err, "error opening file") - return f -} - -/* -use this function as needed to create new test data. - -example: - - func Test_CreateData(t *testing.T) { - createTestData(t, "celestia-node/share/eds/testdata") - } -*/ -func createTestData(t *testing.T, testDir string) { //nolint:unused - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - err := os.Chdir(testDir) - require.NoError(t, err, "changing to the directory") - os.RemoveAll("example.car") - require.NoError(t, err, "removing old file") - f, err := os.OpenFile("example.car", os.O_WRONLY|os.O_CREATE, 0600) - require.NoError(t, err, "opening file") - - eds := edstest.RandEDS(t, 4) - err = WriteEDS(ctx, eds, f) - require.NoError(t, err, "writing EDS to file") - f.Close() - dah, err := share.NewRoot(eds) - require.NoError(t, err) - - header, err := json.MarshalIndent(dah, "", "") - require.NoError(t, err, "marshaling example root") - os.RemoveAll("example-root.json") - require.NoError(t, err, "removing old file") - f, err = os.OpenFile("example-root.json", os.O_WRONLY|os.O_CREATE, 0600) - require.NoError(t, err, "opening file") - _, err = f.Write(header) - require.NoError(t, err, "writing example root to file") - f.Close() -} diff --git a/share/eds/inverted_index.go b/share/eds/inverted_index.go deleted file mode 100644 index 799ab6208d..0000000000 --- a/share/eds/inverted_index.go +++ /dev/null @@ -1,102 +0,0 @@ -package eds - -import ( - "context" - "errors" - "fmt" - "runtime" - - "github.com/dgraph-io/badger/v4/options" - "github.com/filecoin-project/dagstore/index" - "github.com/filecoin-project/dagstore/shard" - ds "github.com/ipfs/go-datastore" - dsbadger "github.com/ipfs/go-ds-badger4" - "github.com/multiformats/go-multihash" -) - -const invertedIndexPath = "/inverted_index/" - -// ErrNotFoundInIndex is returned instead of ErrNotFound if the multihash doesn't exist in the index -var ErrNotFoundInIndex = errors.New("does not exist in index") - -// simpleInvertedIndex is an inverted index that only stores a single shard key per multihash. Its -// implementation is modified from the default upstream implementation in dagstore/index. -type simpleInvertedIndex struct { - ds ds.Batching -} - -// newSimpleInvertedIndex returns a new inverted index that only stores a single shard key per -// multihash. This is because we use badger as a storage backend, so updates are expensive, and we -// don't care which shard is used to serve a cid. -func newSimpleInvertedIndex(storePath string) (*simpleInvertedIndex, error) { - opts := dsbadger.DefaultOptions // this should be copied - // turn off value log GC as we don't use value log - opts.GcInterval = 0 - // use minimum amount of NumLevelZeroTables to trigger L0 compaction faster - opts.NumLevelZeroTables = 1 - // MaxLevels = 8 will allow the db to grow to ~11.1 TiB - opts.MaxLevels = 8 - // inverted index stores unique hash keys, so we don't need to detect conflicts - opts.DetectConflicts = false - // we don't need compression for inverted index as it just hashes - opts.Compression = options.None - compactors := runtime.NumCPU() - if compactors < 2 { - compactors = 2 - } - if compactors > opts.MaxLevels { // ensure there is no more compactors than db table levels - compactors = opts.MaxLevels - } - opts.NumCompactors = compactors - - ds, err := dsbadger.NewDatastore(storePath+invertedIndexPath, &opts) - if err != nil { - return nil, fmt.Errorf("can't open Badger Datastore: %w", err) - } - - return &simpleInvertedIndex{ds: ds}, nil -} - -func (s *simpleInvertedIndex) AddMultihashesForShard( - ctx context.Context, - mhIter index.MultihashIterator, - sk shard.Key, -) error { - // in the original implementation, a mutex is used here to prevent unnecessary updates to the - // key. The amount of extra data produced by this is negligible, and the performance benefits - // from removing the lock are significant (indexing is a hot path during sync). - batch, err := s.ds.Batch(ctx) - if err != nil { - return fmt.Errorf("failed to create ds batch: %w", err) - } - - err = mhIter.ForEach(func(mh multihash.Multihash) error { - key := ds.NewKey(string(mh)) - if err := batch.Put(ctx, key, []byte(sk.String())); err != nil { - return fmt.Errorf("failed to put mh=%s, err=%w", mh, err) - } - return nil - }) - if err != nil { - return fmt.Errorf("failed to add index entry: %w", err) - } - - if err := batch.Commit(ctx); err != nil { - return fmt.Errorf("failed to commit batch: %w", err) - } - return nil -} - -func (s *simpleInvertedIndex) GetShardsForMultihash(ctx context.Context, mh multihash.Multihash) ([]shard.Key, error) { - key := ds.NewKey(string(mh)) - sbz, err := s.ds.Get(ctx, key) - if err != nil { - return nil, errors.Join(ErrNotFoundInIndex, err) - } - - return []shard.Key{shard.KeyFromString(string(sbz))}, nil -} - -func (s *simpleInvertedIndex) close() error { - return s.ds.Close() -} diff --git a/share/eds/inverted_index_test.go b/share/eds/inverted_index_test.go deleted file mode 100644 index e83c2be267..0000000000 --- a/share/eds/inverted_index_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package eds - -import ( - "context" - "testing" - - "github.com/filecoin-project/dagstore/shard" - "github.com/multiformats/go-multihash" - "github.com/stretchr/testify/require" -) - -type mockIterator struct { - mhs []multihash.Multihash -} - -func (m *mockIterator) ForEach(f func(mh multihash.Multihash) error) error { - for _, mh := range m.mhs { - if err := f(mh); err != nil { - return err - } - } - return nil -} - -// TestMultihashesForShard ensures that the inverted index correctly stores a single shard key per -// duplicate multihash -func TestMultihashesForShard(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - mhs := []multihash.Multihash{ - multihash.Multihash("mh1"), - multihash.Multihash("mh2"), - multihash.Multihash("mh3"), - } - - mi := &mockIterator{mhs: mhs} - path := t.TempDir() - invertedIndex, err := newSimpleInvertedIndex(path) - require.NoError(t, err) - - // 1. Add all 3 multihashes to shard1 - err = invertedIndex.AddMultihashesForShard(ctx, mi, shard.KeyFromString("shard1")) - require.NoError(t, err) - shardKeys, err := invertedIndex.GetShardsForMultihash(ctx, mhs[0]) - require.NoError(t, err) - require.Equal(t, []shard.Key{shard.KeyFromString("shard1")}, shardKeys) - - // 2. Add mh1 to shard2, and ensure that mh1 no longer points to shard1 - err = invertedIndex.AddMultihashesForShard(ctx, &mockIterator{mhs: mhs[:1]}, shard.KeyFromString("shard2")) - require.NoError(t, err) - shardKeys, err = invertedIndex.GetShardsForMultihash(ctx, mhs[0]) - require.NoError(t, err) - require.Equal(t, []shard.Key{shard.KeyFromString("shard2")}, shardKeys) -} diff --git a/share/eds/metrics.go b/share/eds/metrics.go deleted file mode 100644 index 0fd6740154..0000000000 --- a/share/eds/metrics.go +++ /dev/null @@ -1,270 +0,0 @@ -package eds - -import ( - "context" - "time" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" - - "github.com/celestiaorg/celestia-node/libs/utils" -) - -const ( - failedKey = "failed" - sizeKey = "eds_size" - - putResultKey = "result" - putOK putResult = "ok" - putExists putResult = "exists" - putFailed putResult = "failed" - - opNameKey = "op" - longOpResultKey = "result" - longOpUnresolved longOpResult = "unresolved" - longOpOK longOpResult = "ok" - longOpFailed longOpResult = "failed" - - dagstoreShardStatusKey = "shard_status" -) - -var meter = otel.Meter("eds_store") - -type putResult string - -type longOpResult string - -type metrics struct { - putTime metric.Float64Histogram - getCARTime metric.Float64Histogram - getCARBlockstoreTime metric.Float64Histogram - getDAHTime metric.Float64Histogram - removeTime metric.Float64Histogram - getTime metric.Float64Histogram - hasTime metric.Float64Histogram - listTime metric.Float64Histogram - - shardFailureCount metric.Int64Counter - - longOpTime metric.Float64Histogram - gcTime metric.Float64Histogram -} - -func (s *Store) WithMetrics() error { - putTime, err := meter.Float64Histogram("eds_store_put_time_histogram", - metric.WithDescription("eds store put time histogram(s)")) - if err != nil { - return err - } - - getCARTime, err := meter.Float64Histogram("eds_store_get_car_time_histogram", - metric.WithDescription("eds store get car time histogram(s)")) - if err != nil { - return err - } - - getCARBlockstoreTime, err := meter.Float64Histogram("eds_store_get_car_blockstore_time_histogram", - metric.WithDescription("eds store get car blockstore time histogram(s)")) - if err != nil { - return err - } - - getDAHTime, err := meter.Float64Histogram("eds_store_get_dah_time_histogram", - metric.WithDescription("eds store get dah time histogram(s)")) - if err != nil { - return err - } - - removeTime, err := meter.Float64Histogram("eds_store_remove_time_histogram", - metric.WithDescription("eds store remove time histogram(s)")) - if err != nil { - return err - } - - getTime, err := meter.Float64Histogram("eds_store_get_time_histogram", - metric.WithDescription("eds store get time histogram(s)")) - if err != nil { - return err - } - - hasTime, err := meter.Float64Histogram("eds_store_has_time_histogram", - metric.WithDescription("eds store has time histogram(s)")) - if err != nil { - return err - } - - listTime, err := meter.Float64Histogram("eds_store_list_time_histogram", - metric.WithDescription("eds store list time histogram(s)")) - if err != nil { - return err - } - - shardFailureCount, err := meter.Int64Counter("eds_store_shard_failure_counter", - metric.WithDescription("eds store OpShardFail counter")) - if err != nil { - return err - } - - longOpTime, err := meter.Float64Histogram("eds_store_long_operation_time_histogram", - metric.WithDescription("eds store long operation time histogram(s)")) - if err != nil { - return err - } - - gcTime, err := meter.Float64Histogram("eds_store_gc_time", - metric.WithDescription("dagstore gc time histogram(s)")) - if err != nil { - return err - } - - dagStoreShards, err := meter.Int64ObservableGauge("eds_store_dagstore_shards", - metric.WithDescription("dagstore amount of shards by status")) - if err != nil { - return err - } - - if err = s.cache.Load().EnableMetrics(); err != nil { - return err - } - - callback := func(_ context.Context, observer metric.Observer) error { - stats := s.dgstr.Stats() - for status, amount := range stats { - observer.ObserveInt64(dagStoreShards, int64(amount), - metric.WithAttributes( - attribute.String(dagstoreShardStatusKey, status.String()), - )) - } - return nil - } - - if _, err := meter.RegisterCallback(callback, dagStoreShards); err != nil { - return err - } - - s.metrics = &metrics{ - putTime: putTime, - getCARTime: getCARTime, - getCARBlockstoreTime: getCARBlockstoreTime, - getDAHTime: getDAHTime, - removeTime: removeTime, - getTime: getTime, - hasTime: hasTime, - listTime: listTime, - shardFailureCount: shardFailureCount, - longOpTime: longOpTime, - gcTime: gcTime, - } - return nil -} - -func (m *metrics) observeGCtime(ctx context.Context, dur time.Duration, failed bool) { - if m == nil { - return - } - ctx = utils.ResetContextOnError(ctx) - m.gcTime.Record(ctx, dur.Seconds(), metric.WithAttributes( - attribute.Bool(failedKey, failed))) -} - -func (m *metrics) observeShardFailure(ctx context.Context, shardKey string) { - if m == nil { - return - } - ctx = utils.ResetContextOnError(ctx) - - m.shardFailureCount.Add(ctx, 1, metric.WithAttributes(attribute.String("shard_key", shardKey))) -} - -func (m *metrics) observePut(ctx context.Context, dur time.Duration, result putResult, size uint) { - if m == nil { - return - } - ctx = utils.ResetContextOnError(ctx) - - m.putTime.Record(ctx, dur.Seconds(), metric.WithAttributes( - attribute.String(putResultKey, string(result)), - attribute.Int(sizeKey, int(size)))) -} - -func (m *metrics) observeLongOp(ctx context.Context, opName string, dur time.Duration, result longOpResult) { - if m == nil { - return - } - ctx = utils.ResetContextOnError(ctx) - - m.longOpTime.Record(ctx, dur.Seconds(), metric.WithAttributes( - attribute.String(opNameKey, opName), - attribute.String(longOpResultKey, string(result)))) -} - -func (m *metrics) observeGetCAR(ctx context.Context, dur time.Duration, failed bool) { - if m == nil { - return - } - ctx = utils.ResetContextOnError(ctx) - - m.getCARTime.Record(ctx, dur.Seconds(), metric.WithAttributes( - attribute.Bool(failedKey, failed))) -} - -func (m *metrics) observeCARBlockstore(ctx context.Context, dur time.Duration, failed bool) { - if m == nil { - return - } - ctx = utils.ResetContextOnError(ctx) - - m.getCARBlockstoreTime.Record(ctx, dur.Seconds(), metric.WithAttributes( - attribute.Bool(failedKey, failed))) -} - -func (m *metrics) observeGetDAH(ctx context.Context, dur time.Duration, failed bool) { - if m == nil { - return - } - ctx = utils.ResetContextOnError(ctx) - - m.getDAHTime.Record(ctx, dur.Seconds(), metric.WithAttributes( - attribute.Bool(failedKey, failed))) -} - -func (m *metrics) observeRemove(ctx context.Context, dur time.Duration, failed bool) { - if m == nil { - return - } - ctx = utils.ResetContextOnError(ctx) - - m.removeTime.Record(ctx, dur.Seconds(), metric.WithAttributes( - attribute.Bool(failedKey, failed))) -} - -func (m *metrics) observeGet(ctx context.Context, dur time.Duration, failed bool) { - if m == nil { - return - } - ctx = utils.ResetContextOnError(ctx) - - m.getTime.Record(ctx, dur.Seconds(), metric.WithAttributes( - attribute.Bool(failedKey, failed))) -} - -func (m *metrics) observeHas(ctx context.Context, dur time.Duration, failed bool) { - if m == nil { - return - } - ctx = utils.ResetContextOnError(ctx) - - m.hasTime.Record(ctx, dur.Seconds(), metric.WithAttributes( - attribute.Bool(failedKey, failed))) -} - -func (m *metrics) observeList(ctx context.Context, dur time.Duration, failed bool) { - if m == nil { - return - } - ctx = utils.ResetContextOnError(ctx) - - m.listTime.Record(ctx, dur.Seconds(), metric.WithAttributes( - attribute.Bool(failedKey, failed))) -} diff --git a/share/eds/ods.go b/share/eds/ods.go deleted file mode 100644 index aa1219d41a..0000000000 --- a/share/eds/ods.go +++ /dev/null @@ -1,98 +0,0 @@ -package eds - -import ( - "bufio" - "bytes" - "encoding/binary" - "errors" - "fmt" - "io" - - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/ipld/go-car" - "github.com/ipld/go-car/util" -) - -// bufferedODSReader will read odsSquareSize amount of leaves from reader into the buffer. -// It exposes the buffer to be read by io.Reader interface implementation -type bufferedODSReader struct { - carReader *bufio.Reader - // current is the amount of CARv1 encoded leaves that have been read from reader. When current - // reaches odsSquareSize, bufferedODSReader will prevent further reads by returning io.EOF - current, odsSquareSize int - buf *bytes.Buffer -} - -// ODSReader reads CARv1 encoded data from io.ReadCloser and limits the reader to the CAR header -// and first quadrant (ODS) -func ODSReader(carReader io.Reader) (io.Reader, error) { - if carReader == nil { - return nil, errors.New("eds: can't create ODSReader over nil reader") - } - - odsR := &bufferedODSReader{ - carReader: bufio.NewReader(carReader), - buf: new(bytes.Buffer), - } - - // first LdRead reads the full CAR header to determine amount of shares in the ODS - data, err := util.LdRead(odsR.carReader) - if err != nil { - return nil, fmt.Errorf("reading header: %v", err) - } - - var header car.CarHeader - err = cbor.DecodeInto(data, &header) - if err != nil { - return nil, fmt.Errorf("invalid header: %w", err) - } - - // car header contains both row roots and col roots which is why - // we divide by 4 to get the ODSWidth - odsWidth := len(header.Roots) / 4 - odsR.odsSquareSize = odsWidth * odsWidth - - // NewCarReader will expect to read the header first, so write it first - return odsR, util.LdWrite(odsR.buf, data) -} - -func (r *bufferedODSReader) Read(p []byte) (n int, err error) { - // read leafs to the buffer until it has sufficient data to fill provided container or full ods is - // read - for r.current < r.odsSquareSize && r.buf.Len() < len(p) { - if err := r.readLeaf(); err != nil { - return 0, err - } - - r.current++ - } - - // read buffer to slice - return r.buf.Read(p) -} - -// readLeaf reads one leaf from reader into bufferedODSReader buffer -func (r *bufferedODSReader) readLeaf() error { - if _, err := r.carReader.Peek(1); err != nil { // no more blocks, likely clean io.EOF - return err - } - - l, err := binary.ReadUvarint(r.carReader) - if err != nil { - if err == io.EOF { - return io.ErrUnexpectedEOF // don't silently pretend this is a clean EOF - } - return err - } - - if l > uint64(util.MaxAllowedSectionSize) { // Don't OOM - return fmt.Errorf("malformed car; header `length`: %v is bigger than %v", l, util.MaxAllowedSectionSize) - } - - buf := make([]byte, 8) - n := binary.PutUvarint(buf, l) - r.buf.Write(buf[:n]) - - _, err = r.buf.ReadFrom(io.LimitReader(r.carReader, int64(l))) - return err -} diff --git a/share/eds/ods_test.go b/share/eds/ods_test.go deleted file mode 100644 index 0f7c69e708..0000000000 --- a/share/eds/ods_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package eds - -import ( - "context" - "io" - "testing" - - "github.com/ipld/go-car" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-node/share" -) - -// TestODSReader ensures that the reader returned from ODSReader is capable of reading the CAR -// header and ODS. -func TestODSReader(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - // launch eds store - edsStore, err := newStore(t) - require.NoError(t, err) - err = edsStore.Start(ctx) - require.NoError(t, err) - - // generate random eds data and put it into the store - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - - // get CAR reader from store - r, err := edsStore.GetCAR(ctx, dah.Hash()) - assert.NoError(t, err) - defer func() { - require.NoError(t, r.Close()) - }() - - // create ODSReader wrapper based on car reader to limit reads to ODS only - odsR, err := ODSReader(r) - assert.NoError(t, err) - - // create CAR reader from ODSReader - carReader, err := car.NewCarReader(odsR) - assert.NoError(t, err) - - // validate ODS could be obtained from reader - for i := 0; i < 4; i++ { - for j := 0; j < 4; j++ { - // pick share from original eds - original := eds.GetCell(uint(i), uint(j)) - - // read block from odsReader based reader - block, err := carReader.Next() - assert.NoError(t, err) - - // check that original data from eds is same as data from reader - assert.Equal(t, original, share.GetData(block.RawData())) - } - } - - // Make sure no excess data is available to get from reader - _, err = carReader.Next() - assert.Error(t, io.EOF, err) -} - -// TestODSReaderReconstruction ensures that the reader returned from ODSReader provides sufficient -// data for EDS reconstruction -func TestODSReaderReconstruction(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - // launch eds store - edsStore, err := newStore(t) - require.NoError(t, err) - err = edsStore.Start(ctx) - require.NoError(t, err) - - // generate random eds data and put it into the store - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - - // get CAR reader from store - r, err := edsStore.GetCAR(ctx, dah.Hash()) - assert.NoError(t, err) - defer func() { - require.NoError(t, r.Close()) - }() - - // create ODSReader wrapper based on car reader to limit reads to ODS only - odsR, err := ODSReader(r) - assert.NoError(t, err) - - // reconstruct EDS from ODSReader - loaded, err := ReadEDS(ctx, odsR, dah.Hash()) - assert.NoError(t, err) - - rowRoots, err := eds.RowRoots() - require.NoError(t, err) - loadedRowRoots, err := loaded.RowRoots() - require.NoError(t, err) - require.Equal(t, rowRoots, loadedRowRoots) - - colRoots, err := eds.ColRoots() - require.NoError(t, err) - loadedColRoots, err := loaded.ColRoots() - require.NoError(t, err) - require.Equal(t, colRoots, loadedColRoots) -} diff --git a/share/eds/retriever_no_race_test.go b/share/eds/retriever_no_race_test.go deleted file mode 100644 index 15c6aa2fc4..0000000000 --- a/share/eds/retriever_no_race_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// go:build !race - -package eds - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/wrapper" - "github.com/celestiaorg/nmt" - "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds/byzantine" - "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/ipld" -) - -func TestRetriever_ByzantineError(t *testing.T) { - const width = 8 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - bserv := ipld.NewMemBlockservice() - shares := edstest.RandEDS(t, width).Flattened() - _, err := ipld.ImportShares(ctx, shares, bserv) - require.NoError(t, err) - - // corrupt shares so that eds erasure coding does not match - copy(shares[14][share.NamespaceSize:], shares[15][share.NamespaceSize:]) - - // import corrupted eds - batchAdder := ipld.NewNmtNodeAdder(ctx, bserv, ipld.MaxSizeBatchOption(width*2)) - attackerEDS, err := rsmt2d.ImportExtendedDataSquare( - shares, - share.DefaultRSMT2DCodec(), - wrapper.NewConstructor(uint64(width), - nmt.NodeVisitor(batchAdder.Visit)), - ) - require.NoError(t, err) - err = batchAdder.Commit() - require.NoError(t, err) - - // ensure we rcv an error - dah, err := da.NewDataAvailabilityHeader(attackerEDS) - require.NoError(t, err) - r := NewRetriever(bserv) - _, err = r.Retrieve(ctx, &dah) - var errByz *byzantine.ErrByzantine - require.ErrorAs(t, err, &errByz) -} diff --git a/share/eds/retriever_quadrant.go b/share/eds/retriever_quadrant.go deleted file mode 100644 index 3d616e9cd4..0000000000 --- a/share/eds/retriever_quadrant.go +++ /dev/null @@ -1,102 +0,0 @@ -package eds - -import ( - "math/rand" - "time" - - "github.com/ipfs/go-cid" - - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/share/ipld" -) - -const ( - // there are always 4 quadrants - numQuadrants = 4 - // blockTime equals to the time with which new blocks are produced in the network. - // TODO(@Wondertan): Here we assume that the block time is a minute, but - // block time is a network wide variable/param that has to be taken from - // a proper place - blockTime = time.Minute -) - -// RetrieveQuadrantTimeout defines how much time Retriever waits before -// starting to retrieve another quadrant. -// -// NOTE: -// - The whole data square must be retrieved in less than block time. -// - We have 4 quadrants from two sources(rows, cols) which equals to 8 in total. -var RetrieveQuadrantTimeout = blockTime / numQuadrants * 2 - -type quadrant struct { - // slice of roots to get shares from - roots []cid.Cid - // Example coordinates(x;y) of each quadrant when fetching from column roots - // ------ ------- - // | Q0 | | Q1 | - // |(0;0)| |(1;0)| - // ------ ------- - // | Q2 | | Q3 | - // |(0;1)| |(1;1)| - // ------ ------- - x, y int - // source defines the axis(Row or Col) to fetch the quadrant from - source rsmt2d.Axis -} - -// newQuadrants constructs a slice of quadrants from DAHeader. -// There are always 4 quadrants per each source (row and col), so 8 in total. -// The ordering of quadrants is random. -func newQuadrants(dah *da.DataAvailabilityHeader) []*quadrant { - // combine all the roots into one slice, so they can be easily accessible by index - daRoots := [][][]byte{ - dah.RowRoots, - dah.ColumnRoots, - } - // create a quadrant slice for each source(row;col) - sources := [][]*quadrant{ - make([]*quadrant, numQuadrants), - make([]*quadrant, numQuadrants), - } - for source, quadrants := range sources { - size, qsize := len(daRoots[source]), len(daRoots[source])/2 - roots := make([]cid.Cid, size) - for i, root := range daRoots[source] { - roots[i] = ipld.MustCidFromNamespacedSha256(root) - } - - for i := range quadrants { - // convert quadrant 1D into into 2D coordinates - x, y := i%2, i/2 - quadrants[i] = &quadrant{ - roots: roots[qsize*y : qsize*(y+1)], - x: x, - y: y, - source: rsmt2d.Axis(source), - } - } - } - quadrants := make([]*quadrant, 0, numQuadrants*2) - for _, qs := range sources { - quadrants = append(quadrants, qs...) - } - // shuffle quadrants to be fetched in random order - rand.Shuffle(len(quadrants), func(i, j int) { quadrants[i], quadrants[j] = quadrants[j], quadrants[i] }) - return quadrants -} - -// pos calculates position of a share in a data square. -func (q *quadrant) pos(rootIdx, cellIdx int) (int, int) { - cellIdx += len(q.roots) * q.x - rootIdx += len(q.roots) * q.y - switch q.source { - case rsmt2d.Row: - return rootIdx, cellIdx - case rsmt2d.Col: - return cellIdx, rootIdx - default: - panic("unknown axis") - } -} diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go deleted file mode 100644 index 95da345d17..0000000000 --- a/share/eds/retriever_test.go +++ /dev/null @@ -1,214 +0,0 @@ -package eds - -import ( - "context" - "errors" - "fmt" - "testing" - "time" - - "github.com/ipfs/boxo/blockservice" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/header/headertest" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds/byzantine" - "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/ipld" - "github.com/celestiaorg/celestia-node/share/sharetest" -) - -func TestRetriever_Retrieve(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - bServ := ipld.NewMemBlockservice() - r := NewRetriever(bServ) - - type test struct { - name string - squareSize int - } - tests := []test{ - {"1x1(min)", 1}, - {"2x2(med)", 2}, - {"4x4(med)", 4}, - {"8x8(med)", 8}, - {"16x16(med)", 16}, - {"32x32(med)", 32}, - {"64x64(med)", 64}, - {"128x128(max)", share.MaxSquareSize}, - } - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - // generate EDS - shares := sharetest.RandShares(t, tc.squareSize*tc.squareSize) - in, err := ipld.AddShares(ctx, shares, bServ) - require.NoError(t, err) - - // limit with timeout, specifically retrieval - ctx, cancel := context.WithTimeout(ctx, time.Minute*5) // the timeout is big for the max size which is long - defer cancel() - - dah, err := da.NewDataAvailabilityHeader(in) - require.NoError(t, err) - out, err := r.Retrieve(ctx, &dah) - require.NoError(t, err) - assert.True(t, in.Equals(out)) - }) - } -} - -// TestRetriever_MultipleRandQuadrants asserts that reconstruction succeeds -// when any three random quadrants requested. -func TestRetriever_MultipleRandQuadrants(t *testing.T) { - RetrieveQuadrantTimeout = time.Millisecond * 500 - const squareSize = 32 - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - bServ := ipld.NewMemBlockservice() - r := NewRetriever(bServ) - - // generate EDS - shares := sharetest.RandShares(t, squareSize*squareSize) - in, err := ipld.AddShares(ctx, shares, bServ) - require.NoError(t, err) - - dah, err := da.NewDataAvailabilityHeader(in) - require.NoError(t, err) - ses, err := r.newSession(ctx, &dah) - require.NoError(t, err) - - // wait until two additional quadrants requested - // this reliably allows us to reproduce the issue - time.Sleep(RetrieveQuadrantTimeout * 2) - // then ensure we have enough shares for reconstruction for slow machines e.g. CI - <-ses.Done() - - _, err = ses.Reconstruct(ctx) - assert.NoError(t, err) -} - -func TestFraudProofValidation(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) - defer t.Cleanup(cancel) - bServ := ipld.NewMemBlockservice() - - odsSize := []int{2, 4, 16, 32, 64, 128} - for _, size := range odsSize { - t.Run(fmt.Sprintf("ods size:%d", size), func(t *testing.T) { - var errByz *byzantine.ErrByzantine - faultHeader, err := generateByzantineError(ctx, t, size, bServ) - require.True(t, errors.As(err, &errByz)) - - p := byzantine.CreateBadEncodingProof([]byte("hash"), faultHeader.Height(), errByz) - err = p.Validate(faultHeader) - require.NoError(t, err) - }) - } -} - -func generateByzantineError( - ctx context.Context, - t *testing.T, - odsSize int, - bServ blockservice.BlockService, -) (*header.ExtendedHeader, error) { - eds := edstest.RandByzantineEDS(t, odsSize) - err := ipld.ImportEDS(ctx, eds, bServ) - require.NoError(t, err) - h := headertest.ExtendedHeaderFromEDS(t, 1, eds) - _, err = NewRetriever(bServ).Retrieve(ctx, h.DAH) - - return h, err -} - -/* -BenchmarkBEFPValidation/ods_size:2 31273 38819 ns/op 68052 B/op 366 allocs/op -BenchmarkBEFPValidation/ods_size:4 14664 80439 ns/op 135892 B/op 894 allocs/op -BenchmarkBEFPValidation/ods_size:16 2850 386178 ns/op 587890 B/op 4945 allocs/op -BenchmarkBEFPValidation/ods_size:32 1399 874490 ns/op 1233399 B/op 11284 allocs/op -BenchmarkBEFPValidation/ods_size:64 619 2047540 ns/op 2578008 B/op 25364 allocs/op -BenchmarkBEFPValidation/ods_size:128 259 4934375 ns/op 5418406 B/op 56345 allocs/op -*/ -func BenchmarkBEFPValidation(b *testing.B) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) - defer b.Cleanup(cancel) - bServ := ipld.NewMemBlockservice() - r := NewRetriever(bServ) - t := &testing.T{} - odsSize := []int{2, 4, 16, 32, 64, 128} - for _, size := range odsSize { - b.Run(fmt.Sprintf("ods size:%d", size), func(b *testing.B) { - b.ResetTimer() - b.StopTimer() - eds := edstest.RandByzantineEDS(t, size) - err := ipld.ImportEDS(ctx, eds, bServ) - require.NoError(t, err) - h := headertest.ExtendedHeaderFromEDS(t, 1, eds) - _, err = r.Retrieve(ctx, h.DAH) - var errByz *byzantine.ErrByzantine - require.ErrorAs(t, err, &errByz) - b.StartTimer() - - for i := 0; i < b.N; i++ { - b.ReportAllocs() - p := byzantine.CreateBadEncodingProof([]byte("hash"), h.Height(), errByz) - err = p.Validate(h) - require.NoError(b, err) - } - }) - } -} - -/* -BenchmarkNewErrByzantineData/ods_size:2 29605 38846 ns/op 49518 B/op 579 allocs/op -BenchmarkNewErrByzantineData/ods_size:4 11380 105302 ns/op 134967 B/op 1571 allocs/op -BenchmarkNewErrByzantineData/ods_size:16 1902 631086 ns/op 830199 B/op 9601 allocs/op -BenchmarkNewErrByzantineData/ods_size:32 756 1530985 ns/op 1985272 B/op 22901 allocs/op -BenchmarkNewErrByzantineData/ods_size:64 340 3445544 ns/op 4767053 B/op 54704 allocs/op -BenchmarkNewErrByzantineData/ods_size:128 132 8740678 ns/op 11991093 B/op 136584 allocs/op -*/ -func BenchmarkNewErrByzantineData(b *testing.B) { - odsSize := []int{2, 4, 16, 32, 64, 128} - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - bServ := ipld.NewMemBlockservice() - r := NewRetriever(bServ) - t := &testing.T{} - for _, size := range odsSize { - b.Run(fmt.Sprintf("ods size:%d", size), func(b *testing.B) { - b.StopTimer() - eds := edstest.RandByzantineEDS(t, size) - err := ipld.ImportEDS(ctx, eds, bServ) - require.NoError(t, err) - h := headertest.ExtendedHeaderFromEDS(t, 1, eds) - ses, err := r.newSession(ctx, h.DAH) - require.NoError(t, err) - - select { - case <-ctx.Done(): - b.Fatal(ctx.Err()) - case <-ses.Done(): - } - - _, err = ses.Reconstruct(ctx) - assert.NoError(t, err) - var errByz *rsmt2d.ErrByzantineData - require.ErrorAs(t, err, &errByz) - b.StartTimer() - - for i := 0; i < b.N; i++ { - err = byzantine.NewErrByzantine(ctx, bServ, h.DAH, errByz) - require.NotNil(t, err) - } - }) - } -} diff --git a/share/eds/store.go b/share/eds/store.go deleted file mode 100644 index 816065909e..0000000000 --- a/share/eds/store.go +++ /dev/null @@ -1,644 +0,0 @@ -package eds - -import ( - "bufio" - "bytes" - "context" - "errors" - "fmt" - "io" - "os" - "sync" - "sync/atomic" - "time" - - "github.com/filecoin-project/dagstore" - "github.com/filecoin-project/dagstore/index" - "github.com/filecoin-project/dagstore/mount" - "github.com/filecoin-project/dagstore/shard" - bstore "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/go-datastore" - carv1 "github.com/ipld/go-car" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/libs/utils" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds/cache" - "github.com/celestiaorg/celestia-node/share/ipld" -) - -const ( - blocksPath = "/blocks/" - indexPath = "/index/" - transientsPath = "/transients/" -) - -var ErrNotFound = errors.New("eds not found in store") - -// Store maintains (via DAGStore) a top-level index enabling granular and efficient random access to -// every share and/or Merkle proof over every registered CARv1 file. The EDSStore provides a custom -// blockstore interface implementation to achieve access. The main use-case is randomized sampling -// over the whole chain of EDS block data and getting data by namespace. -type Store struct { - cancel context.CancelFunc - - dgstr *dagstore.DAGStore - mounts *mount.Registry - - bs *blockstore - cache atomic.Pointer[cache.DoubleCache] - - carIdx index.FullIndexRepo - invertedIdx *simpleInvertedIndex - - basepath string - gcInterval time.Duration - // lastGCResult is only stored on the store for testing purposes. - lastGCResult atomic.Pointer[dagstore.GCResult] - - // stripedLocks is used to synchronize parallel operations - stripedLocks [256]sync.Mutex - shardFailures chan dagstore.ShardResult - - metrics *metrics -} - -// NewStore creates a new EDS Store under the given basepath and datastore. -func NewStore(params *Parameters, basePath string, ds datastore.Batching) (*Store, error) { - if err := params.Validate(); err != nil { - return nil, err - } - - err := setupPath(basePath) - if err != nil { - return nil, fmt.Errorf("failed to setup eds.Store directories: %w", err) - } - - r := mount.NewRegistry() - err = r.Register("fs", &inMemoryOnceMount{}) - if err != nil { - return nil, fmt.Errorf("failed to register memory mount on the registry: %w", err) - } - if err != nil { - return nil, fmt.Errorf("failed to register FS mount on the registry: %w", err) - } - - fsRepo, err := index.NewFSRepo(basePath + indexPath) - if err != nil { - return nil, fmt.Errorf("failed to create index repository: %w", err) - } - - invertedIdx, err := newSimpleInvertedIndex(basePath) - if err != nil { - return nil, fmt.Errorf("failed to create index: %w", err) - } - - failureChan := make(chan dagstore.ShardResult) - dagStore, err := dagstore.NewDAGStore( - dagstore.Config{ - TransientsDir: basePath + transientsPath, - IndexRepo: fsRepo, - Datastore: ds, - MountRegistry: r, - TopLevelIndex: invertedIdx, - FailureCh: failureChan, - }, - ) - if err != nil { - return nil, fmt.Errorf("failed to create DAGStore: %w", err) - } - - recentBlocksCache, err := cache.NewAccessorCache("recent", params.RecentBlocksCacheSize) - if err != nil { - return nil, fmt.Errorf("failed to create recent blocks cache: %w", err) - } - - blockstoreCache, err := cache.NewAccessorCache("blockstore", params.BlockstoreCacheSize) - if err != nil { - return nil, fmt.Errorf("failed to create blockstore cache: %w", err) - } - - store := &Store{ - basepath: basePath, - dgstr: dagStore, - carIdx: fsRepo, - invertedIdx: invertedIdx, - gcInterval: params.GCInterval, - mounts: r, - shardFailures: failureChan, - } - store.bs = newBlockstore(store, ds) - store.cache.Store(cache.NewDoubleCache(recentBlocksCache, blockstoreCache)) - return store, nil -} - -func (s *Store) Start(ctx context.Context) error { - err := s.dgstr.Start(ctx) - if err != nil { - return err - } - // start Store only if DagStore succeeds - runCtx, cancel := context.WithCancel(context.Background()) - s.cancel = cancel - // initialize empty gc result to avoid panic on access - s.lastGCResult.Store(&dagstore.GCResult{ - Shards: make(map[shard.Key]error), - }) - - if s.gcInterval != 0 { - go s.gc(runCtx) - } - - go s.watchForFailures(runCtx) - return nil -} - -// Stop stops the underlying DAGStore. -func (s *Store) Stop(context.Context) error { - defer s.cancel() - if err := s.invertedIdx.close(); err != nil { - return err - } - return s.dgstr.Close() -} - -// gc periodically removes all inactive or errored shards. -func (s *Store) gc(ctx context.Context) { - ticker := time.NewTicker(s.gcInterval) - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - tnow := time.Now() - res, err := s.dgstr.GC(ctx) - s.metrics.observeGCtime(ctx, time.Since(tnow), err != nil) - if err != nil { - log.Errorf("garbage collecting dagstore: %v", err) - return - } - s.lastGCResult.Store(res) - } - } -} - -func (s *Store) watchForFailures(ctx context.Context) { - for { - select { - case <-ctx.Done(): - return - case res := <-s.shardFailures: - log.Errorw("removing shard after failure", "key", res.Key, "err", res.Error) - s.metrics.observeShardFailure(ctx, res.Key.String()) - k := share.MustDataHashFromString(res.Key.String()) - err := s.Remove(ctx, k) - if err != nil { - log.Errorw("failed to remove shard after failure", "key", res.Key, "err", err) - } - } - } -} - -// Put stores the given data square with DataRoot's hash as a key. -// -// The square is verified on the Exchange level, and Put only stores the square, trusting it. -// The resulting file stores all the shares and NMT Merkle Proofs of the EDS. -// Additionally, the file gets indexed s.t. store.Blockstore can access them. -func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.ExtendedDataSquare) error { - ctx, span := tracer.Start(ctx, "store/put", trace.WithAttributes( - attribute.Int("width", int(square.Width())), - )) - - tnow := time.Now() - err := s.put(ctx, root, square) - result := putOK - switch { - case errors.Is(err, dagstore.ErrShardExists): - result = putExists - case err != nil: - result = putFailed - } - utils.SetStatusAndEnd(span, err) - s.metrics.observePut(ctx, time.Since(tnow), result, square.Width()) - return err -} - -func (s *Store) put(ctx context.Context, root share.DataHash, square *rsmt2d.ExtendedDataSquare) (err error) { - lk := &s.stripedLocks[root[len(root)-1]] - lk.Lock() - defer lk.Unlock() - - // if root already exists, short-circuit - if has, _ := s.Has(ctx, root); has { - return dagstore.ErrShardExists - } - - key := root.String() - f, err := os.OpenFile(s.basepath+blocksPath+key, os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return err - } - defer closeAndLog("car file", f) - - // save encoded eds into buffer - mount := &inMemoryOnceMount{ - // TODO: buffer could be pre-allocated with capacity calculated based on eds size. - buf: bytes.NewBuffer(nil), - FileMount: mount.FileMount{Path: s.basepath + blocksPath + key}, - } - err = WriteEDS(ctx, square, mount) - if err != nil { - return fmt.Errorf("failed to write EDS to file: %w", err) - } - - // write whole buffered mount data in one go to optimize i/o - if _, err = mount.WriteTo(f); err != nil { - return fmt.Errorf("failed to write EDS to file: %w", err) - } - - ch := make(chan dagstore.ShardResult, 1) - err = s.dgstr.RegisterShard(ctx, shard.KeyFromString(key), mount, ch, dagstore.RegisterOpts{}) - if err != nil { - return fmt.Errorf("failed to initiate shard registration: %w", err) - } - - var result dagstore.ShardResult - select { - case result = <-ch: - case <-ctx.Done(): - // if the context finished before the result was received, track the result in a separate goroutine - go trackLateResult("put", ch, s.metrics, time.Minute*5) - return ctx.Err() - } - - if result.Error != nil { - return fmt.Errorf("failed to register shard: %w", result.Error) - } - - // the accessor returned in the result will be nil, so the shard needs to be acquired first to - // become available in the cache. It might take some time, and the result should not affect the put - // operation, so do it in a goroutine - // TODO: Ideally, only recent blocks should be put in the cache, but there is no way right now to - // check such a condition. - go func() { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - ac, err := s.cache.Load().First().GetOrLoad(ctx, result.Key, s.getAccessor) - if err != nil { - log.Warnw("unable to put accessor to recent blocks accessors cache", "err", err) - return - } - - // need to close returned accessor to remove the reader reference - if err := ac.Close(); err != nil { - log.Warnw("unable to close accessor after loading", "err", err) - } - }() - - return nil -} - -// waitForResult waits for a result from the res channel for a maximum duration specified by -// maxWait. If the result is not received within the specified duration, it logs an error -// indicating that the parent context has expired and the shard registration is stuck. If a result -// is received, it checks for any error and logs appropriate messages. -func trackLateResult(opName string, res <-chan dagstore.ShardResult, metrics *metrics, maxWait time.Duration) { - tnow := time.Now() - select { - case <-time.After(maxWait): - metrics.observeLongOp(context.Background(), opName, time.Since(tnow), longOpUnresolved) - log.Errorf("parent context is expired, while register shard is stuck for more than %v sec", time.Since(tnow)) - return - case result := <-res: - // don't observe if result was received right after launch of the func - if time.Since(tnow) < time.Second { - return - } - if result.Error != nil { - metrics.observeLongOp(context.Background(), opName, time.Since(tnow), longOpFailed) - log.Errorf("failed to register shard after context expired: %v ago, err: %s", time.Since(tnow), result.Error) - return - } - metrics.observeLongOp(context.Background(), opName, time.Since(tnow), longOpOK) - log.Warnf("parent context expired, but register shard finished with no error,"+ - " after context expired: %v ago", time.Since(tnow)) - return - } -} - -// GetCAR takes a DataRoot and returns a buffered reader to the respective EDS serialized as a -// CARv1 file. -// The Reader strictly reads the CAR header and first quadrant (1/4) of the EDS, omitting all the -// NMT Merkle proofs. Integrity of the store data is not verified. -// -// The shard is cached in the Store, so subsequent calls to GetCAR with the same root will use the -// same reader. The cache is responsible for closing the underlying reader. -func (s *Store) GetCAR(ctx context.Context, root share.DataHash) (io.ReadCloser, error) { - ctx, span := tracer.Start(ctx, "store/get-car") - tnow := time.Now() - r, err := s.getCAR(ctx, root) - s.metrics.observeGetCAR(ctx, time.Since(tnow), err != nil) - utils.SetStatusAndEnd(span, err) - return r, err -} - -func (s *Store) getCAR(ctx context.Context, root share.DataHash) (io.ReadCloser, error) { - key := shard.KeyFromString(root.String()) - accessor, err := s.cache.Load().Get(key) - if err == nil { - return newReadCloser(accessor), nil - } - // If the accessor is not found in the cache, create a new one from dagstore. We don't put the - // accessor in the cache here because getCAR is used by shrex-eds. There is a lower probability, - // compared to other cache put triggers, that the same block will be requested again soon. - shardAccessor, err := s.getAccessor(ctx, key) - if err != nil { - return nil, fmt.Errorf("failed to get accessor: %w", err) - } - - return newReadCloser(shardAccessor), nil -} - -// Blockstore returns an IPFS blockstore providing access to individual shares/nodes of all EDS -// registered on the Store. NOTE: The blockstore does not store whole Celestia Blocks but IPFS -// blocks. We represent `shares` and NMT Merkle proofs as IPFS blocks and IPLD nodes so Bitswap can -// access those. -func (s *Store) Blockstore() bstore.Blockstore { - return s.bs -} - -// CARBlockstore returns an IPFS Blockstore providing access to individual shares/nodes of a -// specific EDS identified by DataHash and registered on the Store. NOTE: The Blockstore does not -// store whole Celestia Blocks but IPFS blocks. We represent `shares` and NMT Merkle proofs as IPFS -// blocks and IPLD nodes so Bitswap can access those. -func (s *Store) CARBlockstore( - ctx context.Context, - root share.DataHash, -) (*BlockstoreCloser, error) { - ctx, span := tracer.Start(ctx, "store/car-blockstore") - tnow := time.Now() - cbs, err := s.carBlockstore(ctx, root) - s.metrics.observeCARBlockstore(ctx, time.Since(tnow), err != nil) - utils.SetStatusAndEnd(span, err) - return cbs, err -} - -func (s *Store) carBlockstore( - ctx context.Context, - root share.DataHash, -) (*BlockstoreCloser, error) { - key := shard.KeyFromString(root.String()) - accessor, err := s.cache.Load().Get(key) - if err == nil { - return blockstoreCloser(accessor) - } - - // if the accessor is not found in the cache, create a new one from dagstore - sa, err := s.getAccessor(ctx, key) - if err != nil { - return nil, fmt.Errorf("failed to get accessor: %w", err) - } - return blockstoreCloser(sa) -} - -// GetDAH returns the DataAvailabilityHeader for the EDS identified by DataHash. -func (s *Store) GetDAH(ctx context.Context, root share.DataHash) (*share.Root, error) { - ctx, span := tracer.Start(ctx, "store/car-dah") - tnow := time.Now() - r, err := s.getDAH(ctx, root) - s.metrics.observeGetDAH(ctx, time.Since(tnow), err != nil) - utils.SetStatusAndEnd(span, err) - return r, err -} - -func (s *Store) getDAH(ctx context.Context, root share.DataHash) (*share.Root, error) { - r, err := s.getCAR(ctx, root) - if err != nil { - return nil, fmt.Errorf("eds/store: failed to get CAR file: %w", err) - } - defer closeAndLog("car reader", r) - - carHeader, err := carv1.ReadHeader(bufio.NewReader(r)) - if err != nil { - return nil, fmt.Errorf("eds/store: failed to read car header: %w", err) - } - - dah := dahFromCARHeader(carHeader) - if !bytes.Equal(dah.Hash(), root) { - return nil, fmt.Errorf("eds/store: content integrity mismatch from CAR for root %x", root) - } - return dah, nil -} - -// dahFromCARHeader returns the DataAvailabilityHeader stored in the CIDs of a CARv1 header. -func dahFromCARHeader(carHeader *carv1.CarHeader) *share.Root { - rootCount := len(carHeader.Roots) - rootBytes := make([][]byte, 0, rootCount) - for _, root := range carHeader.Roots { - rootBytes = append(rootBytes, ipld.NamespacedSha256FromCID(root)) - } - return &share.Root{ - RowRoots: rootBytes[:rootCount/2], - ColumnRoots: rootBytes[rootCount/2:], - } -} - -func (s *Store) getAccessor(ctx context.Context, key shard.Key) (cache.Accessor, error) { - ch := make(chan dagstore.ShardResult, 1) - err := s.dgstr.AcquireShard(ctx, key, ch, dagstore.AcquireOpts{}) - if err != nil { - if errors.Is(err, dagstore.ErrShardUnknown) { - return nil, ErrNotFound - } - return nil, fmt.Errorf("failed to initialize shard acquisition: %w", err) - } - - select { - case res := <-ch: - if res.Error != nil { - return nil, fmt.Errorf("failed to acquire shard: %w", res.Error) - } - return res.Accessor, nil - case <-ctx.Done(): - go trackLateResult("get_shard", ch, s.metrics, time.Minute) - return nil, ctx.Err() - } -} - -// Remove removes EDS from Store by the given share.Root hash and cleans up all -// the indexing. -func (s *Store) Remove(ctx context.Context, root share.DataHash) error { - ctx, span := tracer.Start(ctx, "store/remove") - tnow := time.Now() - err := s.remove(ctx, root) - s.metrics.observeRemove(ctx, time.Since(tnow), err != nil) - utils.SetStatusAndEnd(span, err) - return err -} - -func (s *Store) remove(ctx context.Context, root share.DataHash) (err error) { - key := shard.KeyFromString(root.String()) - // remove open links to accessor from cache - if err := s.cache.Load().Remove(key); err != nil { - log.Warnw("remove accessor from cache", "err", err) - } - ch := make(chan dagstore.ShardResult, 1) - err = s.dgstr.DestroyShard(ctx, key, ch, dagstore.DestroyOpts{}) - if err != nil { - return fmt.Errorf("failed to initiate shard destruction: %w", err) - } - - select { - case result := <-ch: - if result.Error != nil { - return fmt.Errorf("failed to destroy shard: %w", result.Error) - } - case <-ctx.Done(): - go trackLateResult("remove", ch, s.metrics, time.Minute) - return ctx.Err() - } - - dropped, err := s.carIdx.DropFullIndex(key) - if !dropped { - log.Warnf("failed to drop index for %s", key) - } - if err != nil { - return fmt.Errorf("failed to drop index for %s: %w", key, err) - } - - err = os.Remove(s.basepath + blocksPath + root.String()) - if err != nil { - return fmt.Errorf("failed to remove CAR file: %w", err) - } - return nil -} - -// Get reads EDS out of Store by given DataRoot. -// -// It reads only one quadrant(1/4) of the EDS and verifies the integrity of the stored data by -// recomputing it. -func (s *Store) Get(ctx context.Context, root share.DataHash) (*rsmt2d.ExtendedDataSquare, error) { - ctx, span := tracer.Start(ctx, "store/get") - tnow := time.Now() - eds, err := s.get(ctx, root) - s.metrics.observeGet(ctx, time.Since(tnow), err != nil) - utils.SetStatusAndEnd(span, err) - return eds, err -} - -func (s *Store) get(ctx context.Context, root share.DataHash) (eds *rsmt2d.ExtendedDataSquare, err error) { - ctx, span := tracer.Start(ctx, "store/get") - defer func() { - utils.SetStatusAndEnd(span, err) - }() - - r, err := s.getCAR(ctx, root) - if err != nil { - return nil, fmt.Errorf("failed to get CAR file: %w", err) - } - defer closeAndLog("car reader", r) - - eds, err = ReadEDS(ctx, r, root) - if err != nil { - return nil, fmt.Errorf("failed to read EDS from CAR file: %w", err) - } - return eds, nil -} - -// Has checks if EDS exists by the given share.Root hash. -func (s *Store) Has(ctx context.Context, root share.DataHash) (has bool, err error) { - ctx, span := tracer.Start(ctx, "store/has") - tnow := time.Now() - eds, err := s.has(ctx, root) - s.metrics.observeHas(ctx, time.Since(tnow), err != nil) - utils.SetStatusAndEnd(span, err) - return eds, err -} - -func (s *Store) has(_ context.Context, root share.DataHash) (bool, error) { - key := root.String() - info, err := s.dgstr.GetShardInfo(shard.KeyFromString(key)) - switch err { - case nil: - return true, info.Error - case dagstore.ErrShardUnknown: - return false, info.Error - default: - return false, err - } -} - -// List lists all the registered EDSes. -func (s *Store) List() ([]share.DataHash, error) { - ctx, span := tracer.Start(context.Background(), "store/list") - tnow := time.Now() - hashes, err := s.list() - s.metrics.observeList(ctx, time.Since(tnow), err != nil) - utils.SetStatusAndEnd(span, err) - return hashes, err -} - -func (s *Store) list() ([]share.DataHash, error) { - shards := s.dgstr.AllShardsInfo() - hashes := make([]share.DataHash, 0, len(shards)) - for shrd := range shards { - hash := share.MustDataHashFromString(shrd.String()) - hashes = append(hashes, hash) - } - return hashes, nil -} - -func setupPath(basepath string) error { - err := os.MkdirAll(basepath+blocksPath, os.ModePerm) - if err != nil { - return fmt.Errorf("failed to create blocks directory: %w", err) - } - err = os.MkdirAll(basepath+transientsPath, os.ModePerm) - if err != nil { - return fmt.Errorf("failed to create transients directory: %w", err) - } - err = os.MkdirAll(basepath+indexPath, os.ModePerm) - if err != nil { - return fmt.Errorf("failed to create index directory: %w", err) - } - return nil -} - -// inMemoryOnceMount is used to allow reading once from buffer before using main mount.Reader -type inMemoryOnceMount struct { - buf *bytes.Buffer - - readOnce atomic.Bool - mount.FileMount -} - -func (m *inMemoryOnceMount) Fetch(ctx context.Context) (mount.Reader, error) { - if m.buf != nil && !m.readOnce.Swap(true) { - reader := &inMemoryReader{Reader: bytes.NewReader(m.buf.Bytes())} - // release memory for gc, otherwise buffer will stick forever - m.buf = nil - return reader, nil - } - return m.FileMount.Fetch(ctx) -} - -func (m *inMemoryOnceMount) Write(b []byte) (int, error) { - return m.buf.Write(b) -} - -func (m *inMemoryOnceMount) WriteTo(w io.Writer) (int64, error) { - return io.Copy(w, bytes.NewReader(m.buf.Bytes())) -} - -// inMemoryReader extends bytes.Reader to implement mount.Reader interface -type inMemoryReader struct { - *bytes.Reader -} - -// Close allows inMemoryReader to satisfy mount.Reader interface -func (r *inMemoryReader) Close() error { - return nil -} diff --git a/share/eds/store_test.go b/share/eds/store_test.go deleted file mode 100644 index 6bc6972bb4..0000000000 --- a/share/eds/store_test.go +++ /dev/null @@ -1,539 +0,0 @@ -package eds - -import ( - "context" - "io" - "os" - "sync" - "testing" - "time" - - "github.com/filecoin-project/dagstore" - "github.com/filecoin-project/dagstore/shard" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" - dsbadger "github.com/ipfs/go-ds-badger4" - "github.com/ipld/go-car" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds/cache" - "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/ipld" -) - -func TestEDSStore(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - edsStore, err := newStore(t) - require.NoError(t, err) - err = edsStore.Start(ctx) - require.NoError(t, err) - - // PutRegistersShard tests if Put registers the shard on the underlying DAGStore - t.Run("PutRegistersShard", func(t *testing.T) { - eds, dah := randomEDS(t) - - // shard hasn't been registered yet - has, err := edsStore.Has(ctx, dah.Hash()) - assert.False(t, has) - assert.NoError(t, err) - - err = edsStore.Put(ctx, dah.Hash(), eds) - assert.NoError(t, err) - - _, err = edsStore.dgstr.GetShardInfo(shard.KeyFromString(dah.String())) - assert.NoError(t, err) - }) - - // PutIndexesEDS ensures that Putting an EDS indexes it into the car index - t.Run("PutIndexesEDS", func(t *testing.T) { - eds, dah := randomEDS(t) - - stat, _ := edsStore.carIdx.StatFullIndex(shard.KeyFromString(dah.String())) - assert.False(t, stat.Exists) - - err = edsStore.Put(ctx, dah.Hash(), eds) - assert.NoError(t, err) - - stat, err = edsStore.carIdx.StatFullIndex(shard.KeyFromString(dah.String())) - assert.True(t, stat.Exists) - assert.NoError(t, err) - }) - - // GetCAR ensures that the reader returned from GetCAR is capable of reading the CAR header and - // ODS. - t.Run("GetCAR", func(t *testing.T) { - eds, dah := randomEDS(t) - - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - - r, err := edsStore.GetCAR(ctx, dah.Hash()) - assert.NoError(t, err) - defer func() { - require.NoError(t, r.Close()) - }() - carReader, err := car.NewCarReader(r) - assert.NoError(t, err) - - for i := 0; i < 4; i++ { - for j := 0; j < 4; j++ { - original := eds.GetCell(uint(i), uint(j)) - block, err := carReader.Next() - assert.NoError(t, err) - assert.Equal(t, original, share.GetData(block.RawData())) - } - } - }) - - t.Run("item not exist", func(t *testing.T) { - root := share.DataHash{1} - _, err := edsStore.GetCAR(ctx, root) - assert.ErrorIs(t, err, ErrNotFound) - - _, err = edsStore.GetDAH(ctx, root) - assert.ErrorIs(t, err, ErrNotFound) - - _, err = edsStore.CARBlockstore(ctx, root) - assert.ErrorIs(t, err, ErrNotFound) - }) - - t.Run("Remove", func(t *testing.T) { - eds, dah := randomEDS(t) - - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - - // assert that file now exists - _, err = os.Stat(edsStore.basepath + blocksPath + dah.String()) - assert.NoError(t, err) - - // accessor will be registered in cache async on put, so give it some time to settle - time.Sleep(time.Millisecond * 100) - - err = edsStore.Remove(ctx, dah.Hash()) - assert.NoError(t, err) - - // shard should no longer be registered on the dagstore - _, err = edsStore.dgstr.GetShardInfo(shard.KeyFromString(dah.String())) - assert.Error(t, err, "shard not found") - - // shard should have been dropped from the index, which also removes the file under /index/ - indexStat, err := edsStore.carIdx.StatFullIndex(shard.KeyFromString(dah.String())) - assert.NoError(t, err) - assert.False(t, indexStat.Exists) - - // file no longer exists - _, err = os.Stat(edsStore.basepath + blocksPath + dah.String()) - assert.ErrorContains(t, err, "no such file or directory") - }) - - t.Run("Remove after OpShardFail", func(t *testing.T) { - eds, dah := randomEDS(t) - - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - - // assert that shard now exists - ok, err := edsStore.Has(ctx, dah.Hash()) - assert.NoError(t, err) - assert.True(t, ok) - - // assert that file now exists - path := edsStore.basepath + blocksPath + dah.String() - _, err = os.Stat(path) - assert.NoError(t, err) - - err = os.Remove(path) - assert.NoError(t, err) - - // accessor will be registered in cache async on put, so give it some time to settle - time.Sleep(time.Millisecond * 100) - - // remove non-failed accessor from cache - err = edsStore.cache.Load().Remove(shard.KeyFromString(dah.String())) - assert.NoError(t, err) - - _, err = edsStore.GetCAR(ctx, dah.Hash()) - assert.Error(t, err) - - ticker := time.NewTicker(time.Millisecond * 100) - defer ticker.Stop() - for { - select { - case <-ticker.C: - has, err := edsStore.Has(ctx, dah.Hash()) - if err == nil && !has { - // shard no longer exists after OpShardFail was detected from GetCAR call - return - } - case <-ctx.Done(): - t.Fatal("timeout waiting for shard to be removed") - } - } - }) - - t.Run("Has", func(t *testing.T) { - eds, dah := randomEDS(t) - - ok, err := edsStore.Has(ctx, dah.Hash()) - assert.NoError(t, err) - assert.False(t, ok) - - err = edsStore.Put(ctx, dah.Hash(), eds) - assert.NoError(t, err) - - ok, err = edsStore.Has(ctx, dah.Hash()) - assert.NoError(t, err) - assert.True(t, ok) - }) - - t.Run("RecentBlocksCache", func(t *testing.T) { - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - - // accessor will be registered in cache async on put, so give it some time to settle - time.Sleep(time.Millisecond * 100) - - // check, that the key is in the cache after put - shardKey := shard.KeyFromString(dah.String()) - _, err = edsStore.cache.Load().Get(shardKey) - assert.NoError(t, err) - }) - - t.Run("List", func(t *testing.T) { - const amount = 10 - hashes := make([]share.DataHash, 0, amount) - for range make([]byte, amount) { - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - hashes = append(hashes, dah.Hash()) - } - - hashesOut, err := edsStore.List() - require.NoError(t, err) - for _, hash := range hashes { - assert.Contains(t, hashesOut, hash) - } - }) - - t.Run("Parallel put", func(t *testing.T) { - const amount = 20 - eds, dah := randomEDS(t) - - wg := sync.WaitGroup{} - for i := 1; i < amount; i++ { - wg.Add(1) - go func() { - defer wg.Done() - err := edsStore.Put(ctx, dah.Hash(), eds) - if err != nil { - require.ErrorIs(t, err, dagstore.ErrShardExists) - } - }() - } - wg.Wait() - - eds, err := edsStore.Get(ctx, dah.Hash()) - require.NoError(t, err) - newDah, err := da.NewDataAvailabilityHeader(eds) - require.NoError(t, err) - require.Equal(t, dah.Hash(), newDah.Hash()) - }) -} - -// TestEDSStore_GC verifies that unused transient shards are collected by the GC periodically. -func TestEDSStore_GC(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - edsStore, err := newStore(t) - edsStore.gcInterval = time.Second - require.NoError(t, err) - - // kicks off the gc goroutine - err = edsStore.Start(ctx) - require.NoError(t, err) - - eds, dah := randomEDS(t) - shardKey := shard.KeyFromString(dah.String()) - - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - - // accessor will be registered in cache async on put, so give it some time to settle - time.Sleep(time.Millisecond * 100) - - // remove links to the shard from cache - time.Sleep(time.Millisecond * 100) - key := shard.KeyFromString(share.DataHash(dah.Hash()).String()) - err = edsStore.cache.Load().Remove(key) - require.NoError(t, err) - - // doesn't exist yet - assert.NotContains(t, edsStore.lastGCResult.Load().Shards, shardKey) - - // wait for gc to run, retry three times - for i := 0; i < 3; i++ { - time.Sleep(edsStore.gcInterval) - if _, ok := edsStore.lastGCResult.Load().Shards[shardKey]; ok { - break - } - } - assert.Contains(t, edsStore.lastGCResult.Load().Shards, shardKey) - - // assert nil in this context means there was no error re-acquiring the shard during GC - assert.Nil(t, edsStore.lastGCResult.Load().Shards[shardKey]) -} - -func Test_BlockstoreCache(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - edsStore, err := newStore(t) - require.NoError(t, err) - err = edsStore.Start(ctx) - require.NoError(t, err) - - // store eds to the store with noopCache to allow clean cache after put - swap := edsStore.cache.Load() - edsStore.cache.Store(cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{})) - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - - // get any key from saved eds - bs, err := edsStore.carBlockstore(ctx, dah.Hash()) - require.NoError(t, err) - defer func() { - require.NoError(t, bs.Close()) - }() - keys, err := bs.AllKeysChan(ctx) - require.NoError(t, err) - var key cid.Cid - select { - case key = <-keys: - case <-ctx.Done(): - t.Fatal("context timeout") - } - - // swap back original cache - edsStore.cache.Store(swap) - - // key shouldn't be in cache yet, check for returned errCacheMiss - shardKey := shard.KeyFromString(dah.String()) - _, err = edsStore.cache.Load().Get(shardKey) - require.Error(t, err) - - // now get it from blockstore, to trigger storing to cache - _, err = edsStore.Blockstore().Get(ctx, key) - require.NoError(t, err) - - // should be no errCacheMiss anymore - _, err = edsStore.cache.Load().Get(shardKey) - require.NoError(t, err) -} - -// Test_CachedAccessor verifies that the reader represented by a cached accessor can be read from -// multiple times, without exhausting the underlying reader. -func Test_CachedAccessor(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - edsStore, err := newStore(t) - require.NoError(t, err) - err = edsStore.Start(ctx) - require.NoError(t, err) - - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - - // accessor will be registered in cache async on put, so give it some time to settle - time.Sleep(time.Millisecond * 100) - - // accessor should be in cache - _, err = edsStore.cache.Load().Get(shard.KeyFromString(dah.String())) - require.NoError(t, err) - - // first read from cached accessor - carReader, err := edsStore.getCAR(ctx, dah.Hash()) - require.NoError(t, err) - firstBlock, err := io.ReadAll(carReader) - require.NoError(t, err) - require.NoError(t, carReader.Close()) - - // second read from cached accessor - carReader, err = edsStore.getCAR(ctx, dah.Hash()) - require.NoError(t, err) - secondBlock, err := io.ReadAll(carReader) - require.NoError(t, err) - require.NoError(t, carReader.Close()) - - require.Equal(t, firstBlock, secondBlock) -} - -// Test_CachedAccessor verifies that the reader represented by a accessor obtained directly from -// dagstore can be read from multiple times, without exhausting the underlying reader. -func Test_NotCachedAccessor(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - edsStore, err := newStore(t) - require.NoError(t, err) - err = edsStore.Start(ctx) - require.NoError(t, err) - // replace cache with noopCache to - edsStore.cache.Store(cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{})) - - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(t, err) - - // accessor will be registered in cache async on put, so give it some time to settle - time.Sleep(time.Millisecond * 100) - - // accessor should not be in cache - _, err = edsStore.cache.Load().Get(shard.KeyFromString(dah.String())) - require.Error(t, err) - - // first read from direct accessor (not from cache) - carReader, err := edsStore.getCAR(ctx, dah.Hash()) - require.NoError(t, err) - firstBlock, err := io.ReadAll(carReader) - require.NoError(t, err) - require.NoError(t, carReader.Close()) - - // second read from direct accessor (not from cache) - carReader, err = edsStore.getCAR(ctx, dah.Hash()) - require.NoError(t, err) - secondBlock, err := io.ReadAll(carReader) - require.NoError(t, err) - require.NoError(t, carReader.Close()) - - require.Equal(t, firstBlock, secondBlock) -} - -func BenchmarkStore(b *testing.B) { - ctx, cancel := context.WithCancel(context.Background()) - b.Cleanup(cancel) - - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := NewStore(DefaultParameters(), b.TempDir(), ds) - require.NoError(b, err) - err = edsStore.Start(ctx) - require.NoError(b, err) - - // BenchmarkStore/bench_put_128-10 10 3231859283 ns/op (~3sec) - b.Run("bench put 128", func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - // pause the timer for initializing test data - b.StopTimer() - eds := edstest.RandEDS(b, 128) - dah, err := share.NewRoot(eds) - require.NoError(b, err) - b.StartTimer() - - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(b, err) - } - }) - - // BenchmarkStore/bench_read_128-10 14 78970661 ns/op (~70ms) - b.Run("bench read 128", func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - // pause the timer for initializing test data - b.StopTimer() - eds := edstest.RandEDS(b, 128) - dah, err := share.NewRoot(eds) - require.NoError(b, err) - _ = edsStore.Put(ctx, dah.Hash(), eds) - b.StartTimer() - - _, err = edsStore.Get(ctx, dah.Hash()) - require.NoError(b, err) - } - }) -} - -// BenchmarkCacheEviction benchmarks the time it takes to load a block to the cache, when the -// cache size is set to 1. This forces cache eviction on every read. -// BenchmarkCacheEviction-10/128 384 3533586 ns/op (~3ms) -func BenchmarkCacheEviction(b *testing.B) { - const ( - blocks = 4 - size = 128 - ) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - b.Cleanup(cancel) - - dir := b.TempDir() - ds, err := dsbadger.NewDatastore(dir, &dsbadger.DefaultOptions) - require.NoError(b, err) - - newStore := func(params *Parameters) *Store { - edsStore, err := NewStore(params, dir, ds) - require.NoError(b, err) - err = edsStore.Start(ctx) - require.NoError(b, err) - return edsStore - } - edsStore := newStore(DefaultParameters()) - - // generate EDSs and store them - cids := make([]cid.Cid, blocks) - for i := range cids { - eds := edstest.RandEDS(b, size) - dah, err := da.NewDataAvailabilityHeader(eds) - require.NoError(b, err) - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(b, err) - - // store cids for read loop later - cids[i] = ipld.MustCidFromNamespacedSha256(dah.RowRoots[0]) - } - - // restart store to clear cache - require.NoError(b, edsStore.Stop(ctx)) - - // set BlockstoreCacheSize to 1 to force eviction on every read - params := DefaultParameters() - params.BlockstoreCacheSize = 1 - bstore := newStore(params).Blockstore() - - // start benchmark - b.ResetTimer() - for i := 0; i < b.N; i++ { - h := cids[i%blocks] - // every read will trigger eviction - _, err := bstore.Get(ctx, h) - require.NoError(b, err) - } -} - -func newStore(t *testing.T) (*Store, error) { - t.Helper() - - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - return NewStore(DefaultParameters(), t.TempDir(), ds) -} - -func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.Root) { - eds := edstest.RandEDS(t, 4) - dah, err := share.NewRoot(eds) - require.NoError(t, err) - - return eds, dah -} diff --git a/share/eds/testdata/README.md b/share/eds/testdata/README.md deleted file mode 100644 index 960549e2a0..0000000000 --- a/share/eds/testdata/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# CARxEDS Testdata - -This directory contains an example CARv1 file of an EDS and its matching data availability header. - -They might need to be regenerated when modifying constants such as the default share size. This can be done by running the test utility in `eds_test.go` called `createTestData`. diff --git a/share/eds/testdata/example-root.json b/share/eds/testdata/example-root.json deleted file mode 100644 index 999d6301b6..0000000000 --- a/share/eds/testdata/example-root.json +++ /dev/null @@ -1,22 +0,0 @@ -{ -"row_roots": [ -"AAAAAAAAAAAAAAAAAAAAAAAAABPYEuDlO9Dz69oAAAAAAAAAAAAAAAAAAAAAAAAAMcklN0h38T4b/UBC/Cmr5YWmjmmxvi1e35vZBW14b8gDHBoTFVvY6H4J", -"AAAAAAAAAAAAAAAAAAAAAAAAADxyZecUZD41W5IAAAAAAAAAAAAAAAAAAAAAAAAAh8vQUZ38PaWyeUs7dQhphIuRIKiGaTr4KFwEhMRhejTd6/4NHdnKTDyY", -"AAAAAAAAAAAAAAAAAAAAAAAAAKDQatbQSwQ9uJsAAAAAAAAAAAAAAAAAAAAAAAAArtdqXCSsM1OlVCRZqqfZDnEO9eC5cwlgy5MQHb2g4NLr7nZYTruiOoz7", -"AAAAAAAAAAAAAAAAAAAAAAAAAMeUhM8LZBo9sWwAAAAAAAAAAAAAAAAAAAAAAAAA8PtvJpbDc4APKOK6MT1k61HuQXwauWw3nFWwr9pSljiYMv6jjjdLDF8o", -"/////////////////////////////////////////////////////////////////////////////xnHmhDh4Y8vfJrgewAcvLWpvI5XOyATj1IQDkCwvIEh", -"/////////////////////////////////////////////////////////////////////////////+qngp0AfoykfXwsMBukRtYxNA/bzW0+F3J7Q/+S1YZJ", -"/////////////////////////////////////////////////////////////////////////////4WNPrME/2MLrIZgAUoKaVx2GzJqDcYGrBg+sudPKUDy", -"/////////////////////////////////////////////////////////////////////////////6HdebpaHl7iTpLvmuPvtQNnkHfNOPyEhahxbVnIB2d1" -], -"column_roots": [ -"AAAAAAAAAAAAAAAAAAAAAAAAABPYEuDlO9Dz69oAAAAAAAAAAAAAAAAAAAAAAAAAx5SEzwtkGj2xbESyOeamsjGWUBQdAQoiSl+rMtNMo1wEtfGQnFS/g+K+", -"AAAAAAAAAAAAAAAAAAAAAAAAAC3uK6nhCxHTfBwAAAAAAAAAAAAAAAAAAAAAAAAA1fxnqHyO6qV39pcUQ8MuTfJ7RBhbSVWf0aamUP27KRY0II55oJoY6Ng6", -"AAAAAAAAAAAAAAAAAAAAAAAAAC6DkYeeBY/kKvAAAAAAAAAAAAAAAAAAAAAAAAAA47rxk8hoCnWGM+CX47TlYWBeE2unvRhA/j3EvHdxeL1rFRkaYfAd5eg7", -"AAAAAAAAAAAAAAAAAAAAAAAAADHJJTdId/E+G/0AAAAAAAAAAAAAAAAAAAAAAAAA8PtvJpbDc4APKAk5QPSH59HECE2sf/CDLKAZJjWo9DD4sLXJQ4jTZoH6", -"/////////////////////////////////////////////////////////////////////////////4lKCT3K11RnNIuLNfY+SfDZCYAE2iW0hjQHIVBpoN0q", -"/////////////////////////////////////////////////////////////////////////////1NpYcgayEVenbFeEO5LJ1j1/1sD+PvZWHDv+jqT1dLR", -"/////////////////////////////////////////////////////////////////////////////8FOWVuCU0rTzUW9tP2R47RmTBvwXX8ycKrMhgKEi1xa", -"/////////////////////////////////////////////////////////////////////////////7K5SoZ3HF5QgPvIXpKSr9eT4Xfiokc3PUMmXE4pBDTf" -] -} \ No newline at end of file diff --git a/share/eds/testdata/example.car b/share/eds/testdata/example.car deleted file mode 100644 index 4d33c0ef33..0000000000 Binary files a/share/eds/testdata/example.car and /dev/null differ diff --git a/share/eds/utils.go b/share/eds/utils.go deleted file mode 100644 index b897dd14b5..0000000000 --- a/share/eds/utils.go +++ /dev/null @@ -1,152 +0,0 @@ -package eds - -import ( - "context" - "errors" - "fmt" - "io" - - "github.com/filecoin-project/dagstore" - "github.com/ipfs/boxo/blockservice" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - "golang.org/x/sync/errgroup" - - "github.com/celestiaorg/celestia-node/libs/utils" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds/cache" - "github.com/celestiaorg/celestia-node/share/ipld" -) - -// readCloser is a helper struct, that combines io.Reader and io.Closer -type readCloser struct { - io.Reader - io.Closer -} - -// BlockstoreCloser represents a blockstore that can also be closed. It combines the functionality -// of a dagstore.ReadBlockstore with that of an io.Closer. -type BlockstoreCloser struct { - dagstore.ReadBlockstore - io.Closer -} - -func newReadCloser(ac cache.Accessor) io.ReadCloser { - return readCloser{ - ac.Reader(), - ac, - } -} - -// blockstoreCloser constructs new BlockstoreCloser from cache.Accessor -func blockstoreCloser(ac cache.Accessor) (*BlockstoreCloser, error) { - bs, err := ac.Blockstore() - if err != nil { - return nil, fmt.Errorf("eds/store: failed to get blockstore: %w", err) - } - return &BlockstoreCloser{ - ReadBlockstore: bs, - Closer: ac, - }, nil -} - -func closeAndLog(name string, closer io.Closer) { - if err := closer.Close(); err != nil { - log.Warnw("closing "+name, "err", err) - } -} - -// RetrieveNamespaceFromStore gets all EDS shares in the given namespace from -// the EDS store through the corresponding CAR-level blockstore. It is extracted -// from the store getter to make it available for reuse in the shrexnd server. -func RetrieveNamespaceFromStore( - ctx context.Context, - store *Store, - dah *share.Root, - namespace share.Namespace, -) (shares share.NamespacedShares, err error) { - if err = namespace.ValidateForData(); err != nil { - return nil, err - } - - bs, err := store.CARBlockstore(ctx, dah.Hash()) - if errors.Is(err, ErrNotFound) { - // convert error to satisfy getter interface contract - err = share.ErrNotFound - } - if err != nil { - return nil, fmt.Errorf("failed to retrieve blockstore from eds store: %w", err) - } - defer func() { - if err := bs.Close(); err != nil { - log.Warnw("closing blockstore", "err", err) - } - }() - - // wrap the read-only CAR blockstore in a getter - blockGetter := NewBlockGetter(bs) - shares, err = CollectSharesByNamespace(ctx, blockGetter, dah, namespace) - if errors.Is(err, ipld.ErrNodeNotFound) { - // IPLD node not found after the index pointed to this shard and the CAR - // blockstore has been opened successfully is a strong indicator of - // corruption. We remove the block on bridges and fulls and return - // share.ErrNotFound to ensure the data is retrieved by the next getter. - // Note that this recovery is manual and will only be restored by an RPC - // call to SharesAvailable that fetches the same datahash that was - // removed. - err = store.Remove(ctx, dah.Hash()) - if err != nil { - log.Errorf("failed to remove CAR from store after detected corruption: %w", err) - } - err = share.ErrNotFound - } - if err != nil { - return nil, fmt.Errorf("failed to retrieve shares by namespace from store: %w", err) - } - - return shares, nil -} - -// CollectSharesByNamespace collects NamespaceShares within the given namespace from share.Root. -func CollectSharesByNamespace( - ctx context.Context, - bg blockservice.BlockGetter, - root *share.Root, - namespace share.Namespace, -) (shares share.NamespacedShares, err error) { - ctx, span := tracer.Start(ctx, "collect-shares-by-namespace", trace.WithAttributes( - attribute.String("namespace", namespace.String()), - )) - defer func() { - utils.SetStatusAndEnd(span, err) - }() - - rootCIDs := ipld.FilterRootByNamespace(root, namespace) - if len(rootCIDs) == 0 { - return []share.NamespacedRow{}, nil - } - - errGroup, ctx := errgroup.WithContext(ctx) - shares = make([]share.NamespacedRow, len(rootCIDs)) - for i, rootCID := range rootCIDs { - // shadow loop variables, to ensure correct values are captured - i, rootCID := i, rootCID - errGroup.Go(func() error { - row, proof, err := ipld.GetSharesByNamespace(ctx, bg, rootCID, namespace, len(root.RowRoots)) - shares[i] = share.NamespacedRow{ - Shares: row, - Proof: proof, - } - if err != nil { - return fmt.Errorf("retrieving shares by namespace %s for row %x: %w", namespace.String(), rootCID, err) - } - return nil - }) - } - - if err := errGroup.Wait(); err != nil { - return nil, err - } - - return shares, nil -} diff --git a/share/getter.go b/share/getter.go index 3fcc93de33..c121f262c1 100644 --- a/share/getter.go +++ b/share/getter.go @@ -18,6 +18,9 @@ var ( // ErrOutOfBounds is used to indicate that a passed row or column index is out of bounds of the // square size. ErrOutOfBounds = errors.New("share: row or column index is larger than square size") + // ErrOperationNotSupported is used to indicate that the operation is not supported by the + // implementation. + ErrOperationNotSupported = errors.New("operation is not supported") ) // Getter interface provides a set of accessors for shares by the Root. @@ -72,16 +75,19 @@ func (ns NamespacedShares) Verify(root *Root, namespace Namespace) error { } for i, row := range ns { + if row.Proof == nil && row.Shares == nil { + return fmt.Errorf("row verification failed: no proofs and shares") + } // verify row data against row hash from original root - if !row.verify(originalRoots[i], namespace) { + if !row.Verify(originalRoots[i], namespace) { return fmt.Errorf("row verification failed: row %d doesn't match original root: %s", i, root.String()) } } return nil } -// verify validates the row using nmt inclusion proof. -func (row *NamespacedRow) verify(rowRoot []byte, namespace Namespace) bool { +// Verify validates the row using nmt inclusion proof. +func (row *NamespacedRow) Verify(rowRoot []byte, namespace Namespace) bool { // construct nmt leaves from shares by prepending namespace leaves := make([][]byte, 0, len(row.Shares)) for _, shr := range row.Shares { diff --git a/share/getters/cascade.go b/share/getters/cascade.go index 3875127580..42e211e3f7 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -132,7 +132,7 @@ func cascadeGetters[V any]( return val, nil } - if errors.Is(getErr, errOperationNotSupported) { + if errors.Is(getErr, share.ErrOperationNotSupported) { continue } diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 7297766652..0f3795e6c2 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -2,15 +2,9 @@ package getters import ( "context" - "os" - "sync" + "sync/atomic" "testing" - "time" - "github.com/ipfs/boxo/exchange/offline" - "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" - dsbadger "github.com/ipfs/go-ds-badger4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -21,43 +15,40 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/ipld" - "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/celestiaorg/celestia-node/share/store" + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" ) func TestStoreGetter(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - tmpDir := t.TempDir() - storeCfg := eds.DefaultParameters() - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := eds.NewStore(storeCfg, tmpDir, ds) - require.NoError(t, err) - - err = edsStore.Start(ctx) + edsStore, err := store.NewStore(store.DefaultParameters(), t.TempDir()) require.NoError(t, err) sg := NewStoreGetter(edsStore) + height := atomic.Uint64{} t.Run("GetShare", func(t *testing.T) { - randEds, eh := randomEDS(t) - err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) + eds, eh := randomEDS(t) + height := height.Add(1) + eh.RawHeader.Height = int64(height) + f, err := edsStore.Put(ctx, eh.DAH.Hash(), height, eds) require.NoError(t, err) + defer f.Close() - squareSize := int(randEds.Width()) + squareSize := int(eds.Width()) for i := 0; i < squareSize; i++ { for j := 0; j < squareSize; j++ { share, err := sg.GetShare(ctx, eh, i, j) require.NoError(t, err) - assert.Equal(t, randEds.GetCell(uint(i), uint(j)), share) + assert.Equal(t, eds.GetCell(uint(i), uint(j)), share) } } // doesn't panic on indexes too high - _, err := sg.GetShare(ctx, eh, squareSize, squareSize) + _, err = sg.GetShare(ctx, eh, squareSize, squareSize) require.ErrorIs(t, err, share.ErrOutOfBounds) // root not found @@ -67,159 +58,46 @@ func TestStoreGetter(t *testing.T) { }) t.Run("GetEDS", func(t *testing.T) { - randEds, eh := randomEDS(t) - err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) + eds, eh := randomEDS(t) + height := height.Add(1) + eh.RawHeader.Height = int64(height) + f, err := edsStore.Put(ctx, eh.DAH.Hash(), height, eds) require.NoError(t, err) + defer f.Close() retrievedEDS, err := sg.GetEDS(ctx, eh) require.NoError(t, err) - assert.True(t, randEds.Equals(retrievedEDS)) + assert.True(t, eds.Equals(retrievedEDS)) // root not found - emptyRoot := da.MinDataAvailabilityHeader() - eh.DAH = &emptyRoot + eh.RawHeader.Height = 666 _, err = sg.GetEDS(ctx, eh) - require.ErrorIs(t, err, share.ErrNotFound) + require.ErrorIs(t, err, share.ErrNotFound, err) }) - t.Run("GetSharesByNamespace", func(t *testing.T) { - randEds, namespace, eh := randomEDSWithDoubledNamespace(t, 4) - err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) - require.NoError(t, err) - - shares, err := sg.GetSharesByNamespace(ctx, eh, namespace) - require.NoError(t, err) - require.NoError(t, shares.Verify(eh.DAH, namespace)) - assert.Len(t, shares.Flatten(), 2) - - // namespace not found - randNamespace := sharetest.RandV0Namespace() - emptyShares, err := sg.GetSharesByNamespace(ctx, eh, randNamespace) - require.NoError(t, err) - require.Empty(t, emptyShares.Flatten()) - - // root not found + t.Run("Get empty EDS", func(t *testing.T) { + // empty root emptyRoot := da.MinDataAvailabilityHeader() - eh.DAH = &emptyRoot - _, err = sg.GetSharesByNamespace(ctx, eh, namespace) - require.ErrorIs(t, err, share.ErrNotFound) - }) - - t.Run("GetSharesFromNamespace removes corrupted shard", func(t *testing.T) { - randEds, namespace, eh := randomEDSWithDoubledNamespace(t, 4) - err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) - require.NoError(t, err) - - // available - shares, err := sg.GetSharesByNamespace(ctx, eh, namespace) - require.NoError(t, err) - require.NoError(t, shares.Verify(eh.DAH, namespace)) - assert.Len(t, shares.Flatten(), 2) - - // 'corrupt' existing CAR by overwriting with a random EDS - f, err := os.OpenFile(tmpDir+"/blocks/"+eh.DAH.String(), os.O_WRONLY, 0644) - require.NoError(t, err) - edsToOverwriteWith, eh := randomEDS(t) - err = eds.WriteEDS(ctx, edsToOverwriteWith, f) - require.NoError(t, err) - - shares, err = sg.GetSharesByNamespace(ctx, eh, namespace) - require.ErrorIs(t, err, share.ErrNotFound) - require.Nil(t, shares) - - // corruption detected, shard is removed - // try every 200ms until it passes or the context ends - ticker := time.NewTicker(200 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - t.Fatal("context ended before successful retrieval") - case <-ticker.C: - has, err := edsStore.Has(ctx, eh.DAH.Hash()) - if err != nil { - t.Fatal(err) - } - if !has { - require.NoError(t, err) - return - } - } - } - }) -} - -func TestIPLDGetter(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - storeCfg := eds.DefaultParameters() - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := eds.NewStore(storeCfg, t.TempDir(), ds) - require.NoError(t, err) - - err = edsStore.Start(ctx) - require.NoError(t, err) - - bStore := edsStore.Blockstore() - bserv := ipld.NewBlockservice(bStore, offline.Exchange(edsStore.Blockstore())) - sg := NewIPLDGetter(bserv) - - t.Run("GetShare", func(t *testing.T) { - ctx, cancel := context.WithTimeout(ctx, time.Second) - t.Cleanup(cancel) - - randEds, eh := randomEDS(t) - err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) + eh := headertest.RandExtendedHeaderWithRoot(t, &emptyRoot) + f, err := edsStore.Put(ctx, eh.DAH.Hash(), eh.Height(), nil) require.NoError(t, err) + require.NoError(t, f.Close()) - squareSize := int(randEds.Width()) - for i := 0; i < squareSize; i++ { - for j := 0; j < squareSize; j++ { - share, err := sg.GetShare(ctx, eh, i, j) - require.NoError(t, err) - assert.Equal(t, randEds.GetCell(uint(i), uint(j)), share) - } - } - - // doesn't panic on indexes too high - _, err := sg.GetShare(ctx, eh, squareSize+1, squareSize+1) - require.ErrorIs(t, err, share.ErrOutOfBounds) - - // root not found - _, eh = randomEDS(t) - _, err = sg.GetShare(ctx, eh, 0, 0) - require.ErrorIs(t, err, share.ErrNotFound) - }) - - t.Run("GetEDS", func(t *testing.T) { - ctx, cancel := context.WithTimeout(ctx, time.Second) - t.Cleanup(cancel) - - randEds, eh := randomEDS(t) - err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) + eds, err := sg.GetEDS(ctx, eh) require.NoError(t, err) - - retrievedEDS, err := sg.GetEDS(ctx, eh) + dah, err := share.NewRoot(eds) require.NoError(t, err) - assert.True(t, randEds.Equals(retrievedEDS)) - - // Ensure blocks still exist after cleanup - colRoots, _ := retrievedEDS.ColRoots() - has, err := bStore.Has(ctx, ipld.MustCidFromNamespacedSha256(colRoots[0])) - assert.NoError(t, err) - assert.True(t, has) + require.True(t, share.DataHash(dah.Hash()).IsEmptyRoot()) }) t.Run("GetSharesByNamespace", func(t *testing.T) { - ctx, cancel := context.WithTimeout(ctx, time.Second) - t.Cleanup(cancel) - - randEds, namespace, eh := randomEDSWithDoubledNamespace(t, 4) - err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) + eds, namespace, eh := randomEDSWithDoubledNamespace(t, 4) + height := height.Add(1) + eh.RawHeader.Height = int64(height) + f, err := edsStore.Put(ctx, eh.DAH.Hash(), height, eds) require.NoError(t, err) + defer f.Close() - // first check that shares are returned correctly if they exist shares, err := sg.GetSharesByNamespace(ctx, eh, namespace) require.NoError(t, err) require.NoError(t, shares.Verify(eh.DAH, namespace)) @@ -231,86 +109,13 @@ func TestIPLDGetter(t *testing.T) { require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) - // nid doesn't exist in root - emptyRoot := da.MinDataAvailabilityHeader() - eh.DAH = &emptyRoot - emptyShares, err = sg.GetSharesByNamespace(ctx, eh, namespace) - require.NoError(t, err) - require.Empty(t, emptyShares.Flatten()) + // root not found + eh.RawHeader.Height = 666 + _, err = sg.GetSharesByNamespace(ctx, eh, namespace) + require.ErrorIs(t, err, share.ErrNotFound, err) }) } -// BenchmarkIPLDGetterOverBusyCache benchmarks the performance of the IPLDGetter when the -// cache size of the underlying blockstore is less than the number of blocks being requested in -// parallel. This is to ensure performance doesn't degrade when the cache is being frequently -// evicted. -// BenchmarkIPLDGetterOverBusyCache-10/128 1 12460428417 ns/op (~12s) -func BenchmarkIPLDGetterOverBusyCache(b *testing.B) { - const ( - blocks = 10 - size = 128 - ) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - b.Cleanup(cancel) - - dir := b.TempDir() - ds, err := dsbadger.NewDatastore(dir, &dsbadger.DefaultOptions) - require.NoError(b, err) - - newStore := func(params *eds.Parameters) *eds.Store { - edsStore, err := eds.NewStore(params, dir, ds) - require.NoError(b, err) - err = edsStore.Start(ctx) - require.NoError(b, err) - return edsStore - } - edsStore := newStore(eds.DefaultParameters()) - - // generate EDSs and store them - headers := make([]*header.ExtendedHeader, blocks) - for i := range headers { - eds := edstest.RandEDS(b, size) - dah, err := da.NewDataAvailabilityHeader(eds) - require.NoError(b, err) - err = edsStore.Put(ctx, dah.Hash(), eds) - require.NoError(b, err) - - eh := headertest.RandExtendedHeader(b) - eh.DAH = &dah - - // store cids for read loop later - headers[i] = eh - } - - // restart store to clear cache - require.NoError(b, edsStore.Stop(ctx)) - - // set BlockstoreCacheSize to 1 to force eviction on every read - params := eds.DefaultParameters() - params.BlockstoreCacheSize = 1 - edsStore = newStore(params) - bstore := edsStore.Blockstore() - bserv := ipld.NewBlockservice(bstore, offline.Exchange(bstore)) - - // start client - getter := NewIPLDGetter(bserv) - - // request blocks in parallel - b.ResetTimer() - g := sync.WaitGroup{} - g.Add(blocks) - for _, h := range headers { - h := h - go func() { - defer g.Done() - _, err := getter.GetEDS(ctx, h) - require.NoError(b, err) - }() - } - g.Wait() -} - func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *header.ExtendedHeader) { eds := edstest.RandEDS(t, 4) dah, err := share.NewRoot(eds) diff --git a/share/getters/ipld.go b/share/getters/ipld.go deleted file mode 100644 index e9c930248d..0000000000 --- a/share/getters/ipld.go +++ /dev/null @@ -1,165 +0,0 @@ -package getters - -import ( - "context" - "errors" - "fmt" - "sync" - "sync/atomic" - - "github.com/ipfs/boxo/blockservice" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/libs/utils" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/eds/byzantine" - "github.com/celestiaorg/celestia-node/share/ipld" -) - -var _ share.Getter = (*IPLDGetter)(nil) - -// IPLDGetter is a share.Getter that retrieves shares from the bitswap network. Result caching is -// handled by the provided blockservice. A blockservice session will be created for retrieval if the -// passed context is wrapped with WithSession. -type IPLDGetter struct { - rtrv *eds.Retriever - bServ blockservice.BlockService -} - -// NewIPLDGetter creates a new share.Getter that retrieves shares from the bitswap network. -func NewIPLDGetter(bServ blockservice.BlockService) *IPLDGetter { - return &IPLDGetter{ - rtrv: eds.NewRetriever(bServ), - bServ: bServ, - } -} - -// GetShare gets a single share at the given EDS coordinates from the bitswap network. -func (ig *IPLDGetter) GetShare(ctx context.Context, header *header.ExtendedHeader, row, col int) (share.Share, error) { - var err error - ctx, span := tracer.Start(ctx, "ipld/get-share", trace.WithAttributes( - attribute.Int("row", row), - attribute.Int("col", col), - )) - defer func() { - utils.SetStatusAndEnd(span, err) - }() - - dah := header.DAH - upperBound := len(dah.RowRoots) - if row >= upperBound || col >= upperBound { - err := share.ErrOutOfBounds - span.RecordError(err) - return nil, err - } - root, leaf := ipld.Translate(dah, row, col) - - // wrap the blockservice in a session if it has been signaled in the context. - blockGetter := getGetter(ctx, ig.bServ) - s, err := ipld.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) - if errors.Is(err, ipld.ErrNodeNotFound) { - // convert error to satisfy getter interface contract - err = share.ErrNotFound - } - if err != nil { - return nil, fmt.Errorf("getter/ipld: failed to retrieve share: %w", err) - } - - return s, nil -} - -func (ig *IPLDGetter) GetEDS( - ctx context.Context, - header *header.ExtendedHeader, -) (eds *rsmt2d.ExtendedDataSquare, err error) { - ctx, span := tracer.Start(ctx, "ipld/get-eds") - defer func() { - utils.SetStatusAndEnd(span, err) - }() - - // rtrv.Retrieve calls shares.GetShares until enough shares are retrieved to reconstruct the EDS - eds, err = ig.rtrv.Retrieve(ctx, header.DAH) - if errors.Is(err, ipld.ErrNodeNotFound) { - // convert error to satisfy getter interface contract - err = share.ErrNotFound - } - var errByz *byzantine.ErrByzantine - if errors.As(err, &errByz) { - return nil, err - } - if err != nil { - return nil, fmt.Errorf("getter/ipld: failed to retrieve eds: %w", err) - } - return eds, nil -} - -func (ig *IPLDGetter) GetSharesByNamespace( - ctx context.Context, - header *header.ExtendedHeader, - namespace share.Namespace, -) (shares share.NamespacedShares, err error) { - ctx, span := tracer.Start(ctx, "ipld/get-shares-by-namespace", trace.WithAttributes( - attribute.String("namespace", namespace.String()), - )) - defer func() { - utils.SetStatusAndEnd(span, err) - }() - - if err = namespace.ValidateForData(); err != nil { - return nil, err - } - - // wrap the blockservice in a session if it has been signaled in the context. - blockGetter := getGetter(ctx, ig.bServ) - shares, err = eds.CollectSharesByNamespace(ctx, blockGetter, header.DAH, namespace) - if errors.Is(err, ipld.ErrNodeNotFound) { - // convert error to satisfy getter interface contract - err = share.ErrNotFound - } - if err != nil { - return nil, fmt.Errorf("getter/ipld: failed to retrieve shares by namespace: %w", err) - } - return shares, nil -} - -var sessionKey = &session{} - -// session is a struct that can optionally be passed by context to the share.Getter methods using -// WithSession to indicate that a blockservice session should be created. -type session struct { - sync.Mutex - atomic.Pointer[blockservice.Session] - ctx context.Context -} - -// WithSession stores an empty session in the context, indicating that a blockservice session should -// be created. -func WithSession(ctx context.Context) context.Context { - return context.WithValue(ctx, sessionKey, &session{ctx: ctx}) -} - -func getGetter(ctx context.Context, service blockservice.BlockService) blockservice.BlockGetter { - s, ok := ctx.Value(sessionKey).(*session) - if !ok { - return service - } - - val := s.Load() - if val != nil { - return val - } - - s.Lock() - defer s.Unlock() - val = s.Load() - if val == nil { - val = blockservice.NewSession(s.ctx, service) - s.Store(val) - } - return val -} diff --git a/share/getters/shrex.go b/share/getters/shrex.go index 826c6b1a10..9d604c3a75 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -16,7 +16,6 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" @@ -118,7 +117,7 @@ func (sg *ShrexGetter) Stop(ctx context.Context) error { } func (sg *ShrexGetter) GetShare(context.Context, *header.ExtendedHeader, int, int) (share.Share, error) { - return nil, fmt.Errorf("getter/shrex: GetShare %w", errOperationNotSupported) + return nil, fmt.Errorf("getter/shrex: GetShare %w", share.ErrOperationNotSupported) } func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { @@ -146,6 +145,7 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader if getErr != nil { log.Debugw("eds: couldn't find peer", "hash", header.DAH.String(), + "height", header.Height(), "err", getErr, "finished (s)", time.Since(start)) sg.metrics.recordEDSAttempt(ctx, attempt, false) @@ -154,7 +154,7 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader reqStart := time.Now() reqCtx, cancel := ctxWithSplitTimeout(ctx, sg.minAttemptsCount-attempt+1, sg.minRequestTimeout) - eds, getErr := sg.edsClient.RequestEDS(reqCtx, header.DAH.Hash(), peer) + eds, getErr := sg.edsClient.RequestEDS(reqCtx, header.DAH, header.Height(), peer) cancel() switch { case getErr == nil: @@ -177,6 +177,7 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader err = errors.Join(err, getErr) } log.Debugw("eds: request failed", + "height", header.Height(), "hash", header.DAH.String(), "peer", peer.String(), "attempt", attempt, @@ -204,10 +205,11 @@ func (sg *ShrexGetter) GetSharesByNamespace( utils.SetStatusAndEnd(span, err) }() - // verify that the namespace could exist inside the roots before starting network requests + // find rows that contains target namespace dah := header.DAH - roots := ipld.FilterRootByNamespace(dah, namespace) - if len(roots) == 0 { + fromRow, toRow := share.RowRangeForNamespace(dah, namespace) + if fromRow == toRow { + // target namespace is out of bounds of all rows in the EDS return []share.NamespacedRow{}, nil } @@ -222,6 +224,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( if getErr != nil { log.Debugw("nd: couldn't find peer", "hash", dah.String(), + "height", header.Height(), "namespace", namespace.String(), "err", getErr, "finished (s)", time.Since(start)) @@ -231,7 +234,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( reqStart := time.Now() reqCtx, cancel := ctxWithSplitTimeout(ctx, sg.minAttemptsCount-attempt+1, sg.minRequestTimeout) - nd, getErr := sg.ndClient.RequestND(reqCtx, dah, namespace, peer) + nd, getErr := sg.ndClient.RequestND(reqCtx, header.Height(), fromRow, toRow, namespace, peer) cancel() switch { case getErr == nil: @@ -260,7 +263,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( err = errors.Join(err, getErr) } log.Debugw("nd: request failed", - "hash", dah.String(), + "height", header.Height(), "namespace", namespace.String(), "peer", peer.String(), "attempt", attempt, diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index 075735579b..680e1b6c98 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -2,8 +2,7 @@ package getters import ( "context" - "encoding/binary" - "errors" + "sync/atomic" "testing" "time" @@ -13,7 +12,9 @@ import ( "github.com/libp2p/go-libp2p/p2p/net/conngater" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/libs/rand" + "github.com/celestiaorg/celestia-app/pkg/da" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" @@ -21,14 +22,14 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" - "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/celestiaorg/celestia-node/share/store" + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" ) func TestShrexGetter(t *testing.T) { @@ -41,10 +42,9 @@ func TestShrexGetter(t *testing.T) { clHost, srvHost := net.Hosts()[0], net.Hosts()[1] // launch eds store and put test data into it - edsStore, err := newStore(t) - require.NoError(t, err) - err = edsStore.Start(ctx) + edsStore, err := store.NewStore(store.DefaultParameters(), t.TempDir()) require.NoError(t, err) + height := atomic.Uint64{} ndClient, _ := newNDClientServer(ctx, t, edsStore, srvHost, clHost) edsClient, _ := newEDSClientServer(ctx, t, edsStore, srvHost, clHost) @@ -61,13 +61,20 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data + size := 128 namespace := sharetest.RandV0Namespace() - randEDS, dah := edstest.RandEDSWithNamespace(t, namespace, 64) + sqSise := size * size + eds, dah := edstest.RandEDSWithNamespace(t, namespace, sqSise/2+rand.Intn(sqSise/2), size) eh := headertest.RandExtendedHeaderWithRoot(t, dah) - require.NoError(t, edsStore.Put(ctx, dah.Hash(), randEDS)) + height := height.Add(1) + eh.RawHeader.Height = int64(height) + + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + defer f.Close() peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), - Height: 1, + Height: height, }) got, err := getter.GetSharesByNamespace(ctx, eh, namespace) @@ -82,9 +89,11 @@ func TestShrexGetter(t *testing.T) { // generate test data _, dah, namespace := generateTestEDS(t) eh := headertest.RandExtendedHeaderWithRoot(t, dah) + height := height.Add(1) + peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), - Height: 1, + Height: height, }) _, err := getter.GetSharesByNamespace(ctx, eh, namespace) @@ -92,28 +101,41 @@ func TestShrexGetter(t *testing.T) { }) t.Run("ND_namespace_not_included", func(t *testing.T) { - ctx, cancel := context.WithTimeout(ctx, time.Second) + ctx, cancel := context.WithTimeout(ctx, time.Second*444) t.Cleanup(cancel) // generate test data eds, dah, maxNamespace := generateTestEDS(t) eh := headertest.RandExtendedHeaderWithRoot(t, dah) - require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) + height := height.Add(1) + eh.RawHeader.Height = int64(height) + + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), - Height: 1, + Height: height, }) - nID, err := addToNamespace(maxNamespace, -1) + nID, err := maxNamespace.AddInt(-1) require.NoError(t, err) // check for namespace to be between max and min namespace in root require.Len(t, ipld.FilterRootByNamespace(dah, nID), 1) - emptyShares, err := getter.GetSharesByNamespace(ctx, eh, nID) + sgetter := NewStoreGetter(edsStore) + emptyShares, err := sgetter.GetSharesByNamespace(ctx, eh, nID) require.NoError(t, err) // no shares should be returned require.Empty(t, emptyShares.Flatten()) require.Nil(t, emptyShares.Verify(dah, nID)) + + emptyShares1, err := getter.GetSharesByNamespace(ctx, eh, nID) + require.Equal(t, emptyShares.Flatten(), emptyShares1.Flatten()) + require.NoError(t, err) + // no shares should be returned + require.Empty(t, emptyShares1.Flatten()) + require.Nil(t, emptyShares1.Verify(dah, nID)) }) t.Run("ND_namespace_not_in_dah", func(t *testing.T) { @@ -123,13 +145,18 @@ func TestShrexGetter(t *testing.T) { // generate test data eds, dah, maxNamespace := generateTestEDS(t) eh := headertest.RandExtendedHeaderWithRoot(t, dah) - require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) + height := height.Add(1) + eh.RawHeader.Height = int64(height) + + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), - Height: 1, + Height: height, }) - namespace, err := addToNamespace(maxNamespace, 1) + namespace, err := maxNamesapce.AddInt(1) require.NoError(t, err) // check for namespace to be not in root require.Len(t, ipld.FilterRootByNamespace(dah, namespace), 0) @@ -146,17 +173,34 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - randEDS, dah, _ := generateTestEDS(t) + eds, dah, _ := generateTestEDS(t) eh := headertest.RandExtendedHeaderWithRoot(t, dah) - require.NoError(t, edsStore.Put(ctx, dah.Hash(), randEDS)) + height := height.Add(1) + eh.RawHeader.Height = int64(height) + + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), - Height: 1, + Height: height, }) got, err := getter.GetEDS(ctx, eh) require.NoError(t, err) - require.Equal(t, randEDS.Flattened(), got.Flattened()) + require.True(t, got.Equals(eds)) + }) + + t.Run("EDS get empty block", func(t *testing.T) { + // empty root + emptyRoot := da.MinDataAvailabilityHeader() + eh := headertest.RandExtendedHeaderWithRoot(t, &emptyRoot) + + eds, err := getter.GetEDS(ctx, eh) + require.NoError(t, err) + dah, err := share.NewRoot(eds) + require.NoError(t, err) + require.True(t, share.DataHash(dah.Hash()).IsEmptyRoot()) }) t.Run("EDS_ctx_deadline", func(t *testing.T) { @@ -165,9 +209,12 @@ func TestShrexGetter(t *testing.T) { // generate test data _, dah, _ := generateTestEDS(t) eh := headertest.RandExtendedHeaderWithRoot(t, dah) + height := height.Add(1) + eh.RawHeader.Height = int64(height) + peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), - Height: 1, + Height: height, }) cancel() @@ -182,9 +229,12 @@ func TestShrexGetter(t *testing.T) { // generate test data _, dah, _ := generateTestEDS(t) eh := headertest.RandExtendedHeaderWithRoot(t, dah) + height := height.Add(1) + eh.RawHeader.Height = int64(height) + peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), - Height: 1, + Height: height, }) _, err := getter.GetEDS(ctx, eh) @@ -192,13 +242,6 @@ func TestShrexGetter(t *testing.T) { }) } -func newStore(t *testing.T) (*eds.Store, error) { - t.Helper() - - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - return eds.NewStore(eds.DefaultParameters(), t.TempDir(), ds) -} - func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.Root, share.Namespace) { eds := edstest.RandEDS(t, 4) dah, err := share.NewRoot(eds) @@ -229,7 +272,7 @@ func testManager( } func newNDClientServer( - ctx context.Context, t *testing.T, edsStore *eds.Store, srvHost, clHost host.Host, + ctx context.Context, t *testing.T, edsStore *store.Store, srvHost, clHost host.Host, ) (*shrexnd.Client, *shrexnd.Server) { params := shrexnd.DefaultParameters() @@ -249,7 +292,7 @@ func newNDClientServer( } func newEDSClientServer( - ctx context.Context, t *testing.T, edsStore *eds.Store, srvHost, clHost host.Host, + ctx context.Context, t *testing.T, edsStore *store.Store, srvHost, clHost host.Host, ) (*shrexeds.Client, *shrexeds.Server) { params := shrexeds.DefaultParameters() @@ -267,103 +310,3 @@ func newEDSClientServer( require.NoError(t, err) return client, server } - -// addToNamespace adds arbitrary int value to namespace, treating namespace as big-endian -// implementation of int -func addToNamespace(namespace share.Namespace, val int) (share.Namespace, error) { - if val == 0 { - return namespace, nil - } - // Convert the input integer to a byte slice and add it to result slice - result := make([]byte, len(namespace)) - if val > 0 { - binary.BigEndian.PutUint64(result[len(namespace)-8:], uint64(val)) - } else { - binary.BigEndian.PutUint64(result[len(namespace)-8:], uint64(-val)) - } - - // Perform addition byte by byte - var carry int - for i := len(namespace) - 1; i >= 0; i-- { - sum := 0 - if val > 0 { - sum = int(namespace[i]) + int(result[i]) + carry - } else { - sum = int(namespace[i]) - int(result[i]) + carry - } - - switch { - case sum > 255: - carry = 1 - sum -= 256 - case sum < 0: - carry = -1 - sum += 256 - default: - carry = 0 - } - - result[i] = uint8(sum) - } - - // Handle any remaining carry - if carry != 0 { - return nil, errors.New("namespace overflow") - } - - return result, nil -} - -func TestAddToNamespace(t *testing.T) { - testCases := []struct { - name string - value int - input share.Namespace - expected share.Namespace - expectedError error - }{ - { - name: "Positive value addition", - value: 42, - input: share.Namespace{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, - expected: share.Namespace{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x2b}, - expectedError: nil, - }, - { - name: "Negative value addition", - value: -42, - input: share.Namespace{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, - expected: share.Namespace{0x1, 0x1, 0x1, 0x1, 0x1, 0x01, 0x1, 0x1, 0x1, 0x0, 0xd7}, - expectedError: nil, - }, - { - name: "Overflow error", - value: 1, - input: share.Namespace{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, - expected: nil, - expectedError: errors.New("namespace overflow"), - }, - { - name: "Overflow error negative", - value: -1, - input: share.Namespace{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, - expected: nil, - expectedError: errors.New("namespace overflow"), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result, err := addToNamespace(tc.input, tc.value) - if tc.expectedError == nil { - require.NoError(t, err) - require.Equal(t, tc.expected, result) - return - } - require.Error(t, err) - if err.Error() != tc.expectedError.Error() { - t.Errorf("Unexpected error message. Expected: %v, Got: %v", tc.expectedError, err) - } - }) - } -} diff --git a/share/getters/store.go b/share/getters/store.go index d66a057c56..68d1a27d8d 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -13,8 +13,7 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/store" ) var _ share.Getter = (*StoreGetter)(nil) @@ -22,11 +21,11 @@ var _ share.Getter = (*StoreGetter)(nil) // StoreGetter is a share.Getter that retrieves shares from an eds.Store. No results are saved to // the eds.Store after retrieval. type StoreGetter struct { - store *eds.Store + store *store.Store } // NewStoreGetter creates a new share.Getter that retrieves shares from an eds.Store. -func NewStoreGetter(store *eds.Store) *StoreGetter { +func NewStoreGetter(store *store.Store) *StoreGetter { return &StoreGetter{ store: store, } @@ -51,25 +50,19 @@ func (sg *StoreGetter) GetShare(ctx context.Context, header *header.ExtendedHead span.RecordError(err) return nil, err } - root, leaf := ipld.Translate(dah, row, col) - bs, err := sg.store.CARBlockstore(ctx, dah.Hash()) - if errors.Is(err, eds.ErrNotFound) { + + file, err := sg.store.GetByHeight(ctx, header.Height()) + if errors.Is(err, store.ErrNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound } if err != nil { - return nil, fmt.Errorf("getter/store: failed to retrieve blockstore: %w", err) + return nil, fmt.Errorf("getter/store: failed to retrieve file: %w", err) } - defer func() { - if err := bs.Close(); err != nil { - log.Warnw("closing blockstore", "err", err) - } - }() + defer utils.CloseAndLog(log, "file", file) - // wrap the read-only CAR blockstore in a getter - blockGetter := eds.NewBlockGetter(bs) - s, err := ipld.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) - if errors.Is(err, ipld.ErrNodeNotFound) { + sh, err := file.Share(ctx, col, row) + if errors.Is(err, store.ErrNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound } @@ -77,7 +70,7 @@ func (sg *StoreGetter) GetShare(ctx context.Context, header *header.ExtendedHead return nil, fmt.Errorf("getter/store: failed to retrieve share: %w", err) } - return s, nil + return sh.Share, nil } // GetEDS gets the EDS identified by the given root from the EDS store. @@ -89,15 +82,25 @@ func (sg *StoreGetter) GetEDS( utils.SetStatusAndEnd(span, err) }() - data, err = sg.store.Get(ctx, header.DAH.Hash()) - if errors.Is(err, eds.ErrNotFound) { + if header.DAH.IsZero() { + return share.EmptyExtendedDataSquare(), nil + } + + file, err := sg.store.GetByHeight(ctx, header.Height()) + if errors.Is(err, store.ErrNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound } + if err != nil { + return nil, fmt.Errorf("getter/store: failed to retrieve file: %w", err) + } + defer utils.CloseAndLog(log, "file", file) + + eds, err := file.EDS(ctx) if err != nil { return nil, fmt.Errorf("getter/store: failed to retrieve eds: %w", err) } - return data, nil + return eds, nil } // GetSharesByNamespace gets all EDS shares in the given namespace from the EDS store through the @@ -114,9 +117,31 @@ func (sg *StoreGetter) GetSharesByNamespace( utils.SetStatusAndEnd(span, err) }() - ns, err := eds.RetrieveNamespaceFromStore(ctx, sg.store, header.DAH, namespace) + // find rows that contains target namespace + from, to := share.RowRangeForNamespace(header.DAH, namespace) + if from == to { + // target namespace is out of bounds of all rows in the EDS + return share.NamespacedShares{}, nil + } + + file, err := sg.store.GetByHeight(ctx, header.Height()) + if errors.Is(err, store.ErrNotFound) { + // convert error to satisfy getter interface contract + err = share.ErrNotFound + } if err != nil { - return nil, fmt.Errorf("getter/store: %w", err) + return nil, fmt.Errorf("getter/store: failed to retrieve file: %w", err) } - return ns, nil + defer utils.CloseAndLog(log, "file", file) + + shares = make(share.NamespacedShares, 0, to-from+1) + for row := from; row < to; row++ { + data, err := file.Data(ctx, namespace, row) + if err != nil { + return nil, fmt.Errorf("getter/store: failed to retrieve namespcaed data: %w", err) + } + shares = append(shares, data) + } + + return shares, nil } diff --git a/share/getters/testing.go b/share/getters/testing.go index fafeb0541c..0982b64774 100644 --- a/share/getters/testing.go +++ b/share/getters/testing.go @@ -13,7 +13,7 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/testing/edstest" ) // TestGetter provides a testing SingleEDSGetter and the root of the EDS it holds. diff --git a/share/getters/utils.go b/share/getters/utils.go index 2260183b4f..70d44a623b 100644 --- a/share/getters/utils.go +++ b/share/getters/utils.go @@ -12,8 +12,6 @@ import ( var ( tracer = otel.Tracer("share/getters") log = logging.Logger("share/getters") - - errOperationNotSupported = errors.New("operation is not supported") ) // ctxWithSplitTimeout will split timeout stored in context by splitFactor and return the result if diff --git a/share/ipld/blockserv.go b/share/ipld/blockserv.go index 2ed2a21c77..b7a9bf84e9 100644 --- a/share/ipld/blockserv.go +++ b/share/ipld/blockserv.go @@ -9,7 +9,7 @@ import ( ) // NewBlockservice constructs Blockservice for fetching NMTrees. -func NewBlockservice(bs blockstore.Blockstore, exchange exchange.Interface) blockservice.BlockService { +func NewBlockservice(bs blockstore.Blockstore, exchange exchange.SessionExchange) blockservice.BlockService { return blockservice.New(bs, exchange, blockservice.WithAllowlist(defaultAllowlist)) } diff --git a/share/ipld/get.go b/share/ipld/get.go index adf2ffa8c5..09de5c3d23 100644 --- a/share/ipld/get.go +++ b/share/ipld/get.go @@ -11,6 +11,9 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/share" ) @@ -157,9 +160,72 @@ func GetLeaves(ctx context.Context, wg.Wait() } -// GetProof fetches and returns the leaf's Merkle Proof. +// GetSharesWithProofs fetches Merkle proofs for the given shares +// and returns the result as an array of ShareWithProof. +func GetSharesWithProofs( + ctx context.Context, + bGetter blockservice.BlockGetter, + rootHash []byte, + shares [][]byte, + axisType rsmt2d.Axis, +) ([]*share.ShareWithProof, error) { + proofs := make([]*share.ShareWithProof, len(shares)) + for index, share := range shares { + if share != nil { + proof, err := GetShareWithProof(ctx, bGetter, rootHash, index, len(shares), axisType) + if err != nil { + return nil, err + } + proofs[index] = proof + } + } + return proofs, nil +} + +// GetShareWithProof fetches a Merkle proof for the given share +func GetShareWithProof( + ctx context.Context, + bGetter blockservice.BlockGetter, + rootHash []byte, + index, + total int, + axisType rsmt2d.Axis, +) (*share.ShareWithProof, error) { + rootCid := MustCidFromNamespacedSha256(rootHash) + proof := make([]cid.Cid, 0) + // TODO(@vgonkivs): Combine GetLeafData and getProofNodes in one function as the are traversing the same + // tree. Add options that will control what data will be fetched. + leaf, err := GetLeaf(ctx, bGetter, rootCid, index, total) + if err != nil { + return nil, err + } + + nodes, err := getProofNodes(ctx, bGetter, rootCid, proof, index, total) + if err != nil { + return nil, err + } + + return &share.ShareWithProof{ + Share: share.GetData(leaf.RawData()), + Proof: buildProof(nodes, index), + Axis: axisType, + }, nil +} + +func buildProof(proofNodes []cid.Cid, sharePos int) *nmt.Proof { + rangeProofs := make([][]byte, 0, len(proofNodes)) + for i := len(proofNodes) - 1; i >= 0; i-- { + node := NamespacedSha256FromCID(proofNodes[i]) + rangeProofs = append(rangeProofs, node) + } + + proof := nmt.NewInclusionProof(sharePos, sharePos+1, rangeProofs, true) + return &proof +} + +// getProofNodes fetches and returns the leaf's Merkle Proof. // It walks down the IPLD NMT tree until it reaches the leaf and returns collected proof -func GetProof( +func getProofNodes( ctx context.Context, bGetter blockservice.BlockGetter, root cid.Cid, @@ -186,7 +252,7 @@ func GetProof( proof = append(proof, lnks[1].Cid) } else { root, leaf = lnks[1].Cid, leaf-total // otherwise go down the second - proof, err = GetProof(ctx, bGetter, root, proof, leaf, total) + proof, err = getProofNodes(ctx, bGetter, root, proof, leaf, total) if err != nil { return nil, err } @@ -194,7 +260,7 @@ func GetProof( } // recursively walk down through selected children - return GetProof(ctx, bGetter, root, proof, leaf, total) + return getProofNodes(ctx, bGetter, root, proof, leaf, total) } // chanGroup implements an atomic wait group, closing a jobs chan diff --git a/share/ipld/get_shares.go b/share/ipld/get_shares.go index 98db7012b5..2883e62761 100644 --- a/share/ipld/get_shares.go +++ b/share/ipld/get_shares.go @@ -44,12 +44,13 @@ func GetShares(ctx context.Context, bg blockservice.BlockGetter, root cid.Cid, s func GetSharesByNamespace( ctx context.Context, bGetter blockservice.BlockGetter, - root cid.Cid, + root []byte, namespace share.Namespace, maxShares int, ) ([]share.Share, *nmt.Proof, error) { + rootCid := MustCidFromNamespacedSha256(root) data := NewNamespaceData(maxShares, namespace, WithLeaves(), WithProofs()) - err := data.CollectLeavesByNamespace(ctx, bGetter, root) + err := data.CollectLeavesByNamespace(ctx, bGetter, rootCid) if err != nil { return nil, nil, err } diff --git a/share/ipld/get_shares_test.go b/share/ipld/get_shares_test.go index 580efcb69b..16dbaca2a3 100644 --- a/share/ipld/get_shares_test.go +++ b/share/ipld/get_shares_test.go @@ -21,8 +21,7 @@ import ( "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" ) func TestGetShare(t *testing.T) { @@ -175,8 +174,7 @@ func TestGetSharesByNamespace(t *testing.T) { rowRoots, err := eds.RowRoots() require.NoError(t, err) for _, row := range rowRoots { - rcid := MustCidFromNamespacedSha256(row) - rowShares, _, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(rowRoots)) + rowShares, _, err := GetSharesByNamespace(ctx, bServ, row, namespace, len(rowRoots)) if errors.Is(err, ErrNamespaceOutsideRange) { continue } @@ -364,8 +362,7 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { rowRoots, err := eds.RowRoots() require.NoError(t, err) for _, row := range rowRoots { - rcid := MustCidFromNamespacedSha256(row) - rowShares, proof, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(rowRoots)) + rowShares, proof, err := GetSharesByNamespace(ctx, bServ, row, namespace, len(rowRoots)) if namespace.IsOutsideRange(row, row) { require.ErrorIs(t, err, ErrNamespaceOutsideRange) continue @@ -387,7 +384,7 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { sha256.New(), namespace.ToNMT(), leaves, - NamespacedSha256FromCID(rcid)) + row) require.True(t, verified) // verify inclusion @@ -395,7 +392,7 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { sha256.New(), namespace.ToNMT(), rowShares, - NamespacedSha256FromCID(rcid)) + row) require.True(t, verified) } } diff --git a/share/ipld/get_test.go b/share/ipld/get_test.go new file mode 100644 index 0000000000..64b8b3aaab --- /dev/null +++ b/share/ipld/get_test.go @@ -0,0 +1,84 @@ +package ipld + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share/testing/sharetest" +) + +func TestGetProof(t *testing.T) { + const width = 4 + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) + defer cancel() + bServ := NewMemBlockservice() + + shares := sharetest.RandShares(t, width*width) + in, err := AddShares(ctx, shares, bServ) + require.NoError(t, err) + + dah, err := da.NewDataAvailabilityHeader(in) + require.NoError(t, err) + var tests = []struct { + roots [][]byte + axisType rsmt2d.Axis + }{ + { + roots: dah.RowRoots, + axisType: rsmt2d.Row, + }, + { + roots: dah.ColumnRoots, + axisType: rsmt2d.Col, + }, + } + + for i, tt := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + for axisIdx, root := range tt.roots { + for shrIdx := 0; uint(shrIdx) < in.Width(); shrIdx++ { + share, err := GetShareWithProof(ctx, bServ, root, shrIdx, int(in.Width()), tt.axisType) + require.NoError(t, err) + require.True(t, share.Validate(root, shrIdx, axisIdx, int(in.Width()))) + } + } + }) + } +} + +func TestGetSharesProofs(t *testing.T) { + const width = 4 + ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) + defer cancel() + bServ := NewMemBlockservice() + + shares := sharetest.RandShares(t, width*width) + in, err := AddShares(ctx, shares, bServ) + require.NoError(t, err) + + dah, err := da.NewDataAvailabilityHeader(in) + require.NoError(t, err) + for axisIdx, root := range dah.ColumnRoots { + rootCid := MustCidFromNamespacedSha256(root) + data := make([][]byte, 0, in.Width()) + for index := 0; uint(index) < in.Width(); index++ { + node, err := GetLeaf(ctx, bServ, rootCid, index, int(in.Width())) + require.NoError(t, err) + data = append(data, node.RawData()[9:]) + } + + proves, err := GetSharesWithProofs(ctx, bServ, root, data, rsmt2d.Col) + require.NoError(t, err) + for i, proof := range proves { + require.True(t, proof.Validate(root, i, axisIdx, int(in.Width()))) + } + } +} diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go index 7ce52859b2..f5065df224 100644 --- a/share/ipld/nmt_adder.go +++ b/share/ipld/nmt_adder.go @@ -103,13 +103,15 @@ func BatchSize(squareSize int) int { // ProofsAdder is used to collect proof nodes, while traversing merkle tree type ProofsAdder struct { - lock sync.RWMutex - proofs map[cid.Cid][]byte + lock sync.RWMutex + collectShares bool + proofs map[cid.Cid][]byte } // NewProofsAdder creates new instance of ProofsAdder. -func NewProofsAdder(squareSize int) *ProofsAdder { +func NewProofsAdder(squareSize int, collectShares bool) *ProofsAdder { return &ProofsAdder{ + collectShares: collectShares, // preallocate map to fit all inner nodes for given square size proofs: make(map[cid.Cid][]byte, innerNodesAmount(squareSize)), } @@ -156,7 +158,7 @@ func (a *ProofsAdder) VisitFn() nmt.NodeVisitorFn { if len(a.proofs) > 0 { return nil } - return a.visitInnerNodes + return a.visitNodes } // Purge removed proofs from ProofsAdder allowing GC to collect the memory @@ -171,10 +173,13 @@ func (a *ProofsAdder) Purge() { a.proofs = nil } -func (a *ProofsAdder) visitInnerNodes(hash []byte, children ...[]byte) { +func (a *ProofsAdder) visitNodes(hash []byte, children ...[]byte) { switch len(children) { case 1: - break + if a.collectShares { + id := MustCidFromNamespacedSha256(hash) + a.addProof(id, children[0]) + } case 2: id := MustCidFromNamespacedSha256(hash) a.addProof(id, append(children[0], children[1]...)) diff --git a/share/ipld/nmt_test.go b/share/ipld/nmt_test.go index 77268d7112..f70220c01f 100644 --- a/share/ipld/nmt_test.go +++ b/share/ipld/nmt_test.go @@ -10,7 +10,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/rsmt2d" - "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/testing/edstest" ) // TestNamespaceFromCID checks that deriving the Namespaced hash from diff --git a/share/ipld/utils.go b/share/ipld/utils.go index d3e987e7f3..4c4da4b290 100644 --- a/share/ipld/utils.go +++ b/share/ipld/utils.go @@ -1,17 +1,15 @@ package ipld import ( - "github.com/ipfs/go-cid" - "github.com/celestiaorg/celestia-node/share" ) // FilterRootByNamespace returns the row roots from the given share.Root that contain the namespace. -func FilterRootByNamespace(root *share.Root, namespace share.Namespace) []cid.Cid { - rowRootCIDs := make([]cid.Cid, 0, len(root.RowRoots)) +func FilterRootByNamespace(root *share.Root, namespace share.Namespace) [][]byte { + rowRootCIDs := make([][]byte, 0, len(root.RowRoots)) for _, row := range root.RowRoots { if !namespace.IsOutsideRange(row, row) { - rowRootCIDs = append(rowRootCIDs, MustCidFromNamespacedSha256(row)) + rowRootCIDs = append(rowRootCIDs, row) } } return rowRootCIDs diff --git a/share/namespace.go b/share/namespace.go index df4ad74058..7d27b58e9a 100644 --- a/share/namespace.go +++ b/share/namespace.go @@ -2,7 +2,9 @@ package share import ( "bytes" + "encoding/binary" "encoding/hex" + "errors" "fmt" appns "github.com/celestiaorg/celestia-app/pkg/namespace" @@ -182,3 +184,49 @@ func (n Namespace) IsGreater(target Namespace) bool { func (n Namespace) IsGreaterOrEqualThan(target Namespace) bool { return bytes.Compare(n, target) > -1 } + +// AddInt adds arbitrary int value to namespace, treating namespace as big-endian +// implementation of int +func (n Namespace) AddInt(val int) (Namespace, error) { + if val == 0 { + return n, nil + } + // Convert the input integer to a byte slice and add it to result slice + result := make([]byte, len(n)) + if val > 0 { + binary.BigEndian.PutUint64(result[len(n)-8:], uint64(val)) + } else { + binary.BigEndian.PutUint64(result[len(n)-8:], uint64(-val)) + } + + // Perform addition byte by byte + var carry int + for i := len(n) - 1; i >= 0; i-- { + sum := 0 + if val > 0 { + sum = int(n[i]) + int(result[i]) + carry + } else { + sum = int(n[i]) - int(result[i]) + carry + } + + switch { + case sum > 255: + carry = 1 + sum -= 256 + case sum < 0: + carry = -1 + sum += 256 + default: + carry = 0 + } + + result[i] = uint8(sum) + } + + // Handle any remaining carry + if carry != 0 { + return nil, errors.New("n overflow") + } + + return result, nil +} diff --git a/share/namespace_test.go b/share/namespace_test.go index 786441b043..c2d3d6328a 100644 --- a/share/namespace_test.go +++ b/share/namespace_test.go @@ -2,6 +2,7 @@ package share import ( "bytes" + "errors" "testing" "github.com/stretchr/testify/assert" @@ -198,6 +199,60 @@ func TestValidateForBlob(t *testing.T) { } } +func TestAddToNamespace(t *testing.T) { + testCases := []struct { + name string + value int + input Namespace + expected Namespace + expectedError error + }{ + { + name: "Positive value addition", + value: 42, + input: Namespace{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + expected: Namespace{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x2b}, + expectedError: nil, + }, + { + name: "Negative value addition", + value: -42, + input: Namespace{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + expected: Namespace{0x1, 0x1, 0x1, 0x1, 0x1, 0x01, 0x1, 0x1, 0x1, 0x0, 0xd7}, + expectedError: nil, + }, + { + name: "Overflow error", + value: 1, + input: Namespace{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + expected: nil, + expectedError: errors.New("namespace overflow"), + }, + { + name: "Overflow error negative", + value: -1, + input: Namespace{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + expected: nil, + expectedError: errors.New("namespace overflow"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result, err := tc.input.AddInt(tc.value) + if tc.expectedError == nil { + require.NoError(t, err) + require.Equal(t, tc.expected, result) + return + } + require.Error(t, err) + if err.Error() != tc.expectedError.Error() { + t.Errorf("Unexpected error message. Expected: %v, Got: %v", tc.expectedError, err) + } + }) + } +} + func primaryReservedNamespace(lastByte byte) Namespace { result := make([]byte, NamespaceSize) result = append(result, appns.NamespaceVersionZero) diff --git a/share/p2p/shrexeds/client.go b/share/p2p/shrexeds/client.go index 7602bb5fb0..6f2531e902 100644 --- a/share/p2p/shrexeds/client.go +++ b/share/p2p/shrexeds/client.go @@ -1,6 +1,7 @@ package shrexeds import ( + "bytes" "context" "errors" "fmt" @@ -17,9 +18,9 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/p2p" pb "github.com/celestiaorg/celestia-node/share/p2p/shrexeds/pb" + "github.com/celestiaorg/celestia-node/share/store/file" ) // Client is responsible for requesting EDSs for blocksync over the ShrEx/EDS protocol. @@ -47,14 +48,18 @@ func NewClient(params *Parameters, host host.Host) (*Client, error) { // RequestEDS requests the ODS from the given peers and returns the EDS upon success. func (c *Client) RequestEDS( ctx context.Context, - dataHash share.DataHash, + root *share.Root, + height uint64, peer peer.ID, ) (*rsmt2d.ExtendedDataSquare, error) { - eds, err := c.doRequest(ctx, dataHash, peer) + eds, err := c.doRequest(ctx, root, height, peer) if err == nil { return eds, nil } - log.Debugw("client: eds request to peer failed", "peer", peer.String(), "hash", dataHash.String(), "error", err) + log.Debugw("client: eds request to peer failed", + "height", height, + "peer", peer.String(), + "error", err) if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { c.metrics.ObserveRequests(ctx, 1, p2p.StatusTimeout) return nil, err @@ -71,7 +76,7 @@ func (c *Client) RequestEDS( if err != p2p.ErrNotFound { log.Warnw("client: eds request to peer failed", "peer", peer.String(), - "hash", dataHash.String(), + "height", height, "err", err) } @@ -80,27 +85,30 @@ func (c *Client) RequestEDS( func (c *Client) doRequest( ctx context.Context, - dataHash share.DataHash, + root *share.Root, + height uint64, to peer.ID, ) (*rsmt2d.ExtendedDataSquare, error) { streamOpenCtx, cancel := context.WithTimeout(ctx, c.params.ServerReadTimeout) defer cancel() stream, err := c.host.NewStream(streamOpenCtx, to, c.protocolID) if err != nil { - return nil, fmt.Errorf("failed to open stream: %w", err) + return nil, fmt.Errorf("open stream: %w", err) } defer stream.Close() c.setStreamDeadlines(ctx, stream) - req := &pb.EDSRequest{Hash: dataHash} + req := &pb.EDSRequest{Height: height} // request ODS - log.Debugw("client: requesting ods", "hash", dataHash.String(), "peer", to.String()) + log.Debugw("client: requesting ods", + "height", height, + "peer", to.String()) _, err = serde.Write(stream, req) if err != nil { stream.Reset() //nolint:errcheck - return nil, fmt.Errorf("failed to write request to stream: %w", err) + return nil, fmt.Errorf("write request to stream: %w", err) } err = stream.CloseWrite() if err != nil { @@ -121,7 +129,7 @@ func (c *Client) doRequest( return nil, p2p.ErrNotFound } stream.Reset() //nolint:errcheck - return nil, fmt.Errorf("failed to read status from stream: %w", err) + return nil, fmt.Errorf("read status from stream: %w", err) } switch resp.Status { @@ -129,9 +137,9 @@ func (c *Client) doRequest( // reset stream deadlines to original values, since read deadline was changed during status read c.setStreamDeadlines(ctx, stream) // use header and ODS bytes to construct EDS and verify it against dataHash - eds, err := eds.ReadEDS(ctx, stream, dataHash) + eds, err := readEds(ctx, stream, root) if err != nil { - return nil, fmt.Errorf("failed to read eds from ods bytes: %w", err) + return nil, fmt.Errorf("read eds from stream: %w", err) } c.metrics.ObserveRequests(ctx, 1, p2p.StatusSuccess) return eds, nil @@ -149,6 +157,31 @@ func (c *Client) doRequest( } } +func readEds(ctx context.Context, stream network.Stream, root *share.Root) (*rsmt2d.ExtendedDataSquare, error) { + eds, err := file.ReadEds(ctx, stream, len(root.RowRoots)) + if err != nil { + return nil, fmt.Errorf("failed to read eds from ods bytes: %w", err) + } + + // verify that the EDS hash matches the expected hash + newDah, err := share.NewRoot(eds) + if err != nil { + return nil, fmt.Errorf("create new root from eds: %w, size:%v , expectedSize:%v", + err, + eds.Width(), + len(root.RowRoots), + ) + } + if !bytes.Equal(newDah.Hash(), root.Hash()) { + return nil, fmt.Errorf( + "content integrity mismatch: imported root %s doesn't match expected root %s", + share.DataHash(newDah.Hash()), + root.Hash(), + ) + } + return eds, nil +} + func (c *Client) setStreamDeadlines(ctx context.Context, stream network.Stream) { // set read/write deadline to use context deadline if it exists if dl, ok := ctx.Deadline(); ok { diff --git a/share/p2p/shrexeds/exchange_test.go b/share/p2p/shrexeds/exchange_test.go index 9155be6dec..b97aa3ac73 100644 --- a/share/p2p/shrexeds/exchange_test.go +++ b/share/p2p/shrexeds/exchange_test.go @@ -6,59 +6,59 @@ import ( "testing" "time" - "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" libhost "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/atomic" + + "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/p2p" + "github.com/celestiaorg/celestia-node/share/store" + "github.com/celestiaorg/celestia-node/share/testing/edstest" ) func TestExchange_RequestEDS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) store, client, server := makeExchange(t) - - err := store.Start(ctx) + err := server.Start(ctx) require.NoError(t, err) - err = server.Start(ctx) - require.NoError(t, err) + height := atomic.NewUint64(1) // Testcase: EDS is immediately available t.Run("EDS_Available", func(t *testing.T) { - eds := edstest.RandEDS(t, 4) - dah, err := share.NewRoot(eds) - require.NoError(t, err) - err = store.Put(ctx, dah.Hash(), eds) + eds, root := testData(t) + height := height.Add(1) + f, err := store.Put(ctx, root.Hash(), height, eds) require.NoError(t, err) + require.NoError(t, f.Close()) - requestedEDS, err := client.RequestEDS(ctx, dah.Hash(), server.host.ID()) + requestedEDS, err := client.RequestEDS(ctx, root, height, server.host.ID()) assert.NoError(t, err) assert.Equal(t, eds.Flattened(), requestedEDS.Flattened()) }) // Testcase: EDS is unavailable initially, but is found after multiple requests t.Run("EDS_AvailableAfterDelay", func(t *testing.T) { - eds := edstest.RandEDS(t, 4) - dah, err := share.NewRoot(eds) - require.NoError(t, err) + eds, root := testData(t) + height := height.Add(1) lock := make(chan struct{}) go func() { <-lock - err = store.Put(ctx, dah.Hash(), eds) + f, err := store.Put(ctx, root.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) require.NoError(t, err) lock <- struct{}{} }() - requestedEDS, err := client.RequestEDS(ctx, dah.Hash(), server.host.ID()) + requestedEDS, err := client.RequestEDS(ctx, root, height, server.host.ID()) assert.ErrorIs(t, err, p2p.ErrNotFound) assert.Nil(t, requestedEDS) @@ -67,34 +67,22 @@ func TestExchange_RequestEDS(t *testing.T) { // wait for write to finish <-lock - requestedEDS, err = client.RequestEDS(ctx, dah.Hash(), server.host.ID()) + requestedEDS, err = client.RequestEDS(ctx, root, height, server.host.ID()) assert.NoError(t, err) assert.Equal(t, eds.Flattened(), requestedEDS.Flattened()) }) - // Testcase: Invalid request excludes peer from round-robin, stopping request - t.Run("EDS_InvalidRequest", func(t *testing.T) { - dataHash := []byte("invalid") - requestedEDS, err := client.RequestEDS(ctx, dataHash, server.host.ID()) - assert.ErrorContains(t, err, "stream reset") - assert.Nil(t, requestedEDS) - }) - t.Run("EDS_err_not_found", func(t *testing.T) { - timeoutCtx, cancel := context.WithTimeout(ctx, time.Second) - t.Cleanup(cancel) - eds := edstest.RandEDS(t, 4) - dah, err := share.NewRoot(eds) + _, root := testData(t) + height := height.Add(1) require.NoError(t, err) - _, err = client.RequestEDS(timeoutCtx, dah.Hash(), server.host.ID()) + _, err = client.RequestEDS(ctx, root, height, server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) }) // Testcase: Concurrency limit reached t.Run("EDS_concurrency_limit", func(t *testing.T) { - store, client, server := makeExchange(t) - - require.NoError(t, store.Start(ctx)) + _, client, server := makeExchange(t) require.NoError(t, server.Start(ctx)) ctx, cancel := context.WithTimeout(ctx, time.Second) @@ -120,29 +108,20 @@ func TestExchange_RequestEDS(t *testing.T) { middleware.RateLimitHandler(mockHandler)) // take server concurrency slots with blocked requests + height := height.Add(1) for i := 0; i < rateLimit; i++ { go func(i int) { - client.RequestEDS(ctx, nil, server.host.ID()) //nolint:errcheck + client.RequestEDS(ctx, nil, height, server.host.ID()) //nolint:errcheck }(i) } // wait until all server slots are taken wg.Wait() - _, err = client.RequestEDS(ctx, nil, server.host.ID()) + _, err = client.RequestEDS(ctx, nil, height, server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) }) } -func newStore(t *testing.T) *eds.Store { - t.Helper() - - storeCfg := eds.DefaultParameters() - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - store, err := eds.NewStore(storeCfg, t.TempDir(), ds) - require.NoError(t, err) - return store -} - func createMocknet(t *testing.T, amount int) []libhost.Host { t.Helper() @@ -152,9 +131,11 @@ func createMocknet(t *testing.T, amount int) []libhost.Host { return net.Hosts() } -func makeExchange(t *testing.T) (*eds.Store, *Client, *Server) { +func makeExchange(t *testing.T) (*store.Store, *Client, *Server) { t.Helper() - store := newStore(t) + cfg := store.DefaultParameters() + store, err := store.NewStore(cfg, t.TempDir()) + require.NoError(t, err) hosts := createMocknet(t, 2) client, err := NewClient(DefaultParameters(), hosts[0]) @@ -164,3 +145,10 @@ func makeExchange(t *testing.T) (*eds.Store, *Client, *Server) { return store, client, server } + +func testData(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.Root) { + eds := edstest.RandEDS(t, 4) + dah, err := share.NewRoot(eds) + require.NoError(t, err) + return eds, dah +} diff --git a/share/p2p/shrexeds/params.go b/share/p2p/shrexeds/params.go index 795cb313ed..d2adad2930 100644 --- a/share/p2p/shrexeds/params.go +++ b/share/p2p/shrexeds/params.go @@ -8,7 +8,7 @@ import ( "github.com/celestiaorg/celestia-node/share/p2p" ) -const protocolString = "/shrex/eds/v0.0.1" +const protocolString = "/shrex/eds/v0.0.2" var log = logging.Logger("shrex/eds") diff --git a/share/p2p/shrexeds/pb/extended_data_square.pb.go b/share/p2p/shrexeds/pb/extended_data_square.pb.go index ed1a96ae3b..12b592afdc 100644 --- a/share/p2p/shrexeds/pb/extended_data_square.pb.go +++ b/share/p2p/shrexeds/pb/extended_data_square.pb.go @@ -54,7 +54,7 @@ func (Status) EnumDescriptor() ([]byte, []int) { } type EDSRequest struct { - Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` + Height uint64 `protobuf:"varint,1,opt,name=height,proto3" json:"height,omitempty"` } func (m *EDSRequest) Reset() { *m = EDSRequest{} } @@ -90,11 +90,11 @@ func (m *EDSRequest) XXX_DiscardUnknown() { var xxx_messageInfo_EDSRequest proto.InternalMessageInfo -func (m *EDSRequest) GetHash() []byte { +func (m *EDSRequest) GetHeight() uint64 { if m != nil { - return m.Hash + return m.Height } - return nil + return 0 } type EDSResponse struct { @@ -152,22 +152,22 @@ func init() { } var fileDescriptor_49d42aa96098056e = []byte{ - // 227 bytes of a gzipped FileDescriptorProto + // 229 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x32, 0x28, 0xce, 0x48, 0x2c, 0x4a, 0xd5, 0x2f, 0x30, 0x2a, 0xd0, 0x2f, 0xce, 0x28, 0x4a, 0xad, 0x48, 0x4d, 0x29, 0xd6, 0x2f, 0x48, 0xd2, 0x4f, 0xad, 0x28, 0x49, 0xcd, 0x4b, 0x49, 0x4d, 0x89, 0x4f, 0x49, 0x2c, 0x49, 0x8c, - 0x2f, 0x2e, 0x2c, 0x4d, 0x2c, 0x4a, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x57, 0x52, 0xe0, 0xe2, - 0x72, 0x75, 0x09, 0x0e, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x12, 0xe2, 0x62, 0xc9, 0x48, - 0x2c, 0xce, 0x90, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x02, 0xb3, 0x95, 0xf4, 0xb8, 0xb8, 0xc1, - 0x2a, 0x8a, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0xe4, 0xb9, 0xd8, 0x8a, 0x4b, 0x12, 0x4b, 0x4a, - 0x8b, 0xc1, 0x8a, 0xf8, 0x8c, 0xd8, 0xf5, 0x82, 0xc1, 0xdc, 0x20, 0xa8, 0xb0, 0x96, 0x15, 0x17, - 0x1b, 0x44, 0x44, 0x88, 0x9b, 0x8b, 0xdd, 0xd3, 0x2f, 0xcc, 0xd1, 0xc7, 0xd3, 0x45, 0x80, 0x41, - 0x88, 0x8d, 0x8b, 0xc9, 0xdf, 0x5b, 0x80, 0x51, 0x88, 0x97, 0x8b, 0xd3, 0xcf, 0x3f, 0x24, 0xde, - 0xcd, 0x3f, 0xd4, 0xcf, 0x45, 0x80, 0x49, 0x88, 0x87, 0x8b, 0xc3, 0xd3, 0x2f, 0xc4, 0x35, 0xc8, - 0xcf, 0xd1, 0x47, 0x80, 0xd9, 0x49, 0xe2, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4, 0x18, 0x1f, - 0x3c, 0x92, 0x63, 0x9c, 0xf0, 0x58, 0x8e, 0xe1, 0xc2, 0x63, 0x39, 0x86, 0x1b, 0x8f, 0xe5, 0x18, - 0x92, 0xd8, 0xc0, 0xce, 0x35, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0x7b, 0x1d, 0xd4, 0xa7, 0xe2, - 0x00, 0x00, 0x00, + 0x2f, 0x2e, 0x2c, 0x4d, 0x2c, 0x4a, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x57, 0x52, 0xe1, 0xe2, + 0x72, 0x75, 0x09, 0x0e, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x12, 0xe3, 0x62, 0xcb, 0x48, + 0xcd, 0x4c, 0xcf, 0x28, 0x91, 0x60, 0x54, 0x60, 0xd4, 0x60, 0x09, 0x82, 0xf2, 0x94, 0xf4, 0xb8, + 0xb8, 0xc1, 0xaa, 0x8a, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0xe4, 0xb9, 0xd8, 0x8a, 0x4b, 0x12, + 0x4b, 0x4a, 0x8b, 0xc1, 0xca, 0xf8, 0x8c, 0xd8, 0xf5, 0x82, 0xc1, 0xdc, 0x20, 0xa8, 0xb0, 0x96, + 0x15, 0x17, 0x1b, 0x44, 0x44, 0x88, 0x9b, 0x8b, 0xdd, 0xd3, 0x2f, 0xcc, 0xd1, 0xc7, 0xd3, 0x45, + 0x80, 0x41, 0x88, 0x8d, 0x8b, 0xc9, 0xdf, 0x5b, 0x80, 0x51, 0x88, 0x97, 0x8b, 0xd3, 0xcf, 0x3f, + 0x24, 0xde, 0xcd, 0x3f, 0xd4, 0xcf, 0x45, 0x80, 0x49, 0x88, 0x87, 0x8b, 0xc3, 0xd3, 0x2f, 0xc4, + 0x35, 0xc8, 0xcf, 0xd1, 0x47, 0x80, 0xd9, 0x49, 0xe2, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4, + 0x18, 0x1f, 0x3c, 0x92, 0x63, 0x9c, 0xf0, 0x58, 0x8e, 0xe1, 0xc2, 0x63, 0x39, 0x86, 0x1b, 0x8f, + 0xe5, 0x18, 0x92, 0xd8, 0xc0, 0x4e, 0x36, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0xc0, 0x6e, 0x10, + 0xf2, 0xe6, 0x00, 0x00, 0x00, } func (m *EDSRequest) Marshal() (dAtA []byte, err error) { @@ -190,12 +190,10 @@ func (m *EDSRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Hash) > 0 { - i -= len(m.Hash) - copy(dAtA[i:], m.Hash) - i = encodeVarintExtendedDataSquare(dAtA, i, uint64(len(m.Hash))) + if m.Height != 0 { + i = encodeVarintExtendedDataSquare(dAtA, i, uint64(m.Height)) i-- - dAtA[i] = 0xa + dAtA[i] = 0x8 } return len(dAtA) - i, nil } @@ -245,9 +243,8 @@ func (m *EDSRequest) Size() (n int) { } var l int _ = l - l = len(m.Hash) - if l > 0 { - n += 1 + l + sovExtendedDataSquare(uint64(l)) + if m.Height != 0 { + n += 1 + sovExtendedDataSquare(uint64(m.Height)) } return n } @@ -300,10 +297,10 @@ func (m *EDSRequest) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Height", wireType) } - var byteLen int + m.Height = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowExtendedDataSquare @@ -313,26 +310,11 @@ func (m *EDSRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + m.Height |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { - return ErrInvalidLengthExtendedDataSquare - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthExtendedDataSquare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Hash = append(m.Hash[:0], dAtA[iNdEx:postIndex]...) - if m.Hash == nil { - m.Hash = []byte{} - } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipExtendedDataSquare(dAtA[iNdEx:]) diff --git a/share/p2p/shrexeds/pb/extended_data_square.proto b/share/p2p/shrexeds/pb/extended_data_square.proto index 63750962e9..636d01697a 100644 --- a/share/p2p/shrexeds/pb/extended_data_square.proto +++ b/share/p2p/shrexeds/pb/extended_data_square.proto @@ -1,7 +1,7 @@ syntax = "proto3"; message EDSRequest { - bytes hash = 1; // identifies the requested EDS. + uint64 height = 1; // identifies the requested EDS. } enum Status { diff --git a/share/p2p/shrexeds/server.go b/share/p2p/shrexeds/server.go index 11b99a3438..b38f3a1728 100644 --- a/share/p2p/shrexeds/server.go +++ b/share/p2p/shrexeds/server.go @@ -14,10 +14,11 @@ import ( "github.com/celestiaorg/go-libp2p-messenger/serde" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share/p2p" p2p_pb "github.com/celestiaorg/celestia-node/share/p2p/shrexeds/pb" + "github.com/celestiaorg/celestia-node/share/store" + "github.com/celestiaorg/celestia-node/share/store/file" ) // Server is responsible for serving ODSs for blocksync over the ShrEx/EDS protocol. @@ -28,7 +29,7 @@ type Server struct { host host.Host protocolID protocol.ID - store *eds.Store + store *store.Store params *Parameters middleware *p2p.Middleware @@ -36,7 +37,7 @@ type Server struct { } // NewServer creates a new ShrEx/EDS server. -func NewServer(params *Parameters, host host.Host, store *eds.Store) (*Server, error) { +func NewServer(params *Parameters, host host.Host, store *store.Store) (*Server, error) { if err := params.Validate(); err != nil { return nil, fmt.Errorf("shrex-eds: server creation failed: %w", err) } @@ -83,15 +84,7 @@ func (s *Server) handleStream(stream network.Stream) { return } - // ensure the requested dataHash is a valid root - hash := share.DataHash(req.Hash) - err = hash.Validate() - if err != nil { - logger.Warnw("server: invalid request", "err", err) - stream.Reset() //nolint:errcheck - return - } - logger = logger.With("hash", hash.String()) + logger = logger.With("height", req.Height) ctx, cancel := context.WithTimeout(s.ctx, s.params.HandleRequestTimeout) defer cancel() @@ -99,22 +92,18 @@ func (s *Server) handleStream(stream network.Stream) { // determine whether the EDS is available in our store // we do not close the reader, so that other requests will not need to re-open the file. // closing is handled by the LRU cache. - edsReader, err := s.store.GetCAR(ctx, hash) + file, err := s.store.GetByHeight(ctx, req.Height) var status p2p_pb.Status switch { case err == nil: - defer func() { - if err := edsReader.Close(); err != nil { - log.Warnw("closing car reader", "err", err) - } - }() + defer utils.CloseAndLog(logger, "file", file) status = p2p_pb.Status_OK - case errors.Is(err, eds.ErrNotFound): - logger.Warnw("server: request hash not found") + case errors.Is(err, store.ErrNotFound): + logger.Warnw("server: request height not found") s.metrics.ObserveRequests(ctx, 1, p2p.StatusNotFound) status = p2p_pb.Status_NOT_FOUND case err != nil: - logger.Errorw("server: get CAR", "err", err) + logger.Errorw("server: get file", "err", err) status = p2p_pb.Status_INTERNAL } @@ -135,7 +124,7 @@ func (s *Server) handleStream(stream network.Stream) { } // start streaming the ODS to the client - err = s.writeODS(logger, edsReader, stream) + err = s.writeODS(logger, file, stream) if err != nil { logger.Warnw("server: writing ods to stream", "err", err) stream.Reset() //nolint:errcheck @@ -179,21 +168,22 @@ func (s *Server) writeStatus(logger *zap.SugaredLogger, status p2p_pb.Status, st return err } -func (s *Server) writeODS(logger *zap.SugaredLogger, edsReader io.Reader, stream network.Stream) error { - err := stream.SetWriteDeadline(time.Now().Add(s.params.ServerWriteTimeout)) +func (s *Server) writeODS(logger *zap.SugaredLogger, file file.EdsFile, stream network.Stream) error { + reader, err := file.Reader() if err != nil { - logger.Debugw("server: set read deadline", "err", err) + return fmt.Errorf("getting ODS reader: %w", err) } - - odsReader, err := eds.ODSReader(edsReader) + err = stream.SetWriteDeadline(time.Now().Add(s.params.ServerWriteTimeout)) if err != nil { - return fmt.Errorf("creating ODS reader: %w", err) + logger.Debugw("server: set read deadline", "err", err) } + buf := make([]byte, s.params.BufferSize) - _, err = io.CopyBuffer(stream, odsReader, buf) + n, err := io.CopyBuffer(stream, reader, buf) if err != nil { - return fmt.Errorf("writing ODS bytes: %w", err) + return fmt.Errorf("written: %v, writing ODS bytes: %w", n, err) } + logger.Debugw("server: wrote ODS", "bytes", n) return nil } diff --git a/share/p2p/shrexnd/client.go b/share/p2p/shrexnd/client.go index 86c5150095..5c1b5407ae 100644 --- a/share/p2p/shrexnd/client.go +++ b/share/p2p/shrexnd/client.go @@ -48,7 +48,8 @@ func NewClient(params *Parameters, host host.Host) (*Client, error) { // Returns NamespacedShares with unverified inclusion proofs against the share.Root. func (c *Client) RequestND( ctx context.Context, - root *share.Root, + height uint64, + fromRow, toRow int, namespace share.Namespace, peer peer.ID, ) (share.NamespacedShares, error) { @@ -56,7 +57,7 @@ func (c *Client) RequestND( return nil, err } - shares, err := c.doRequest(ctx, root, namespace, peer) + shares, err := c.doRequest(ctx, height, fromRow, toRow, namespace, peer) if err == nil { return shares, nil } @@ -81,7 +82,8 @@ func (c *Client) RequestND( func (c *Client) doRequest( ctx context.Context, - root *share.Root, + height uint64, + fromRow, toRow int, namespace share.Namespace, peerID peer.ID, ) (share.NamespacedShares, error) { @@ -94,8 +96,10 @@ func (c *Client) doRequest( c.setStreamDeadlines(ctx, stream) req := &pb.GetSharesByNamespaceRequest{ - RootHash: root.Hash(), + Height: height, Namespace: namespace, + FromRow: uint32(fromRow), + ToRow: uint32(toRow), } _, err = serde.Write(stream, req) diff --git a/share/p2p/shrexnd/exchange_test.go b/share/p2p/shrexnd/exchange_test.go index cb8bbe9d74..923113205b 100644 --- a/share/p2p/shrexnd/exchange_test.go +++ b/share/p2p/shrexnd/exchange_test.go @@ -6,34 +6,30 @@ import ( "testing" "time" - "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" libhost "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/p2p" - "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/celestiaorg/celestia-node/share/store" + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" ) func TestExchange_RequestND_NotFound(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) t.Cleanup(cancel) edsStore, client, server := makeExchange(t) - require.NoError(t, edsStore.Start(ctx)) require.NoError(t, server.Start(ctx)) - t.Run("CAR_not_exist", func(t *testing.T) { + t.Run("File not exist", func(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - root := share.Root{} namespace := sharetest.RandV0Namespace() - _, err := client.RequestND(ctx, &root, namespace, server.host.ID()) + _, err := client.RequestND(ctx, 666, 1, 2, namespace, server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) }) @@ -43,11 +39,15 @@ func TestExchange_RequestND_NotFound(t *testing.T) { eds := edstest.RandEDS(t, 4) dah, err := share.NewRoot(eds) + height := uint64(42) require.NoError(t, err) - require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) namespace := sharetest.RandV0Namespace() - emptyShares, err := client.RequestND(ctx, dah, namespace, server.host.ID()) + fromRow, toRow := share.RowRangeForNamespace(dah, namespace) + emptyShares, err := client.RequestND(ctx, height, fromRow, toRow, namespace, server.host.ID()) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) }) @@ -90,23 +90,22 @@ func TestExchange_RequestND(t *testing.T) { // take server concurrency slots with blocked requests for i := 0; i < rateLimit; i++ { go func(i int) { - client.RequestND(ctx, nil, sharetest.RandV0Namespace(), server.host.ID()) //nolint:errcheck + client.RequestND(ctx, 1, 1, 2, sharetest.RandV0Namespace(), server.host.ID()) //nolint:errcheck }(i) } // wait until all server slots are taken wg.Wait() - _, err = client.RequestND(ctx, nil, sharetest.RandV0Namespace(), server.host.ID()) + _, err = client.RequestND(ctx, 1, 1, 2, sharetest.RandV0Namespace(), server.host.ID()) require.ErrorIs(t, err, p2p.ErrRateLimited) }) } -func newStore(t *testing.T) *eds.Store { +func newStore(t *testing.T) *store.Store { t.Helper() - storeCfg := eds.DefaultParameters() - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - store, err := eds.NewStore(storeCfg, t.TempDir(), ds) + storeCfg := store.DefaultParameters() + store, err := store.NewStore(storeCfg, t.TempDir()) require.NoError(t, err) return store } @@ -120,7 +119,7 @@ func createMocknet(t *testing.T, amount int) []libhost.Host { return net.Hosts() } -func makeExchange(t *testing.T) (*eds.Store, *Client, *Server) { +func makeExchange(t *testing.T) (*store.Store, *Client, *Server) { t.Helper() store := newStore(t) hosts := createMocknet(t, 2) diff --git a/share/p2p/shrexnd/params.go b/share/p2p/shrexnd/params.go index 8489627a07..921999372f 100644 --- a/share/p2p/shrexnd/params.go +++ b/share/p2p/shrexnd/params.go @@ -8,7 +8,7 @@ import ( "github.com/celestiaorg/celestia-node/share/p2p" ) -const protocolString = "/shrex/nd/v0.0.3" +const protocolString = "/shrex/nd/v0.0.4" var log = logging.Logger("shrex/nd") diff --git a/share/p2p/shrexnd/pb/share.pb.go b/share/p2p/shrexnd/pb/share.pb.go index 7e3c11416f..ea510234e2 100644 --- a/share/p2p/shrexnd/pb/share.pb.go +++ b/share/p2p/shrexnd/pb/share.pb.go @@ -55,8 +55,10 @@ func (StatusCode) EnumDescriptor() ([]byte, []int) { } type GetSharesByNamespaceRequest struct { - RootHash []byte `protobuf:"bytes,1,opt,name=root_hash,json=rootHash,proto3" json:"root_hash,omitempty"` + Height uint64 `protobuf:"varint,1,opt,name=height,proto3" json:"height,omitempty"` Namespace []byte `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` + FromRow uint32 `protobuf:"varint,3,opt,name=fromRow,proto3" json:"fromRow,omitempty"` + ToRow uint32 `protobuf:"varint,4,opt,name=toRow,proto3" json:"toRow,omitempty"` } func (m *GetSharesByNamespaceRequest) Reset() { *m = GetSharesByNamespaceRequest{} } @@ -92,11 +94,11 @@ func (m *GetSharesByNamespaceRequest) XXX_DiscardUnknown() { var xxx_messageInfo_GetSharesByNamespaceRequest proto.InternalMessageInfo -func (m *GetSharesByNamespaceRequest) GetRootHash() []byte { +func (m *GetSharesByNamespaceRequest) GetHeight() uint64 { if m != nil { - return m.RootHash + return m.Height } - return nil + return 0 } func (m *GetSharesByNamespaceRequest) GetNamespace() []byte { @@ -106,6 +108,20 @@ func (m *GetSharesByNamespaceRequest) GetNamespace() []byte { return nil } +func (m *GetSharesByNamespaceRequest) GetFromRow() uint32 { + if m != nil { + return m.FromRow + } + return 0 +} + +func (m *GetSharesByNamespaceRequest) GetToRow() uint32 { + if m != nil { + return m.ToRow + } + return 0 +} + type GetSharesByNamespaceStatusResponse struct { Status StatusCode `protobuf:"varint,1,opt,name=status,proto3,enum=share.p2p.shrex.nd.StatusCode" json:"status,omitempty"` } @@ -212,28 +228,30 @@ func init() { func init() { proto.RegisterFile("share/p2p/shrexnd/pb/share.proto", fileDescriptor_ed9f13149b0de397) } var fileDescriptor_ed9f13149b0de397 = []byte{ - // 326 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0x4f, 0x4b, 0xf3, 0x40, - 0x10, 0xc6, 0x93, 0x96, 0x37, 0x6f, 0x3b, 0xad, 0x35, 0x2c, 0x22, 0xc5, 0xca, 0x52, 0x02, 0x42, - 0xf1, 0xb0, 0x81, 0x08, 0x1e, 0x85, 0xd6, 0xfa, 0xa7, 0x58, 0x52, 0xd9, 0xb6, 0xe2, 0x41, 0x28, - 0x1b, 0xbb, 0x92, 0x8b, 0xd9, 0x35, 0xbb, 0x45, 0xfd, 0x16, 0x7e, 0x2c, 0x8f, 0x3d, 0x7a, 0x94, - 0xf6, 0x8b, 0x48, 0xb6, 0xd1, 0x1c, 0xf4, 0xb6, 0xf3, 0xcc, 0x33, 0xbf, 0x7d, 0x66, 0xa0, 0xad, - 0x62, 0x96, 0x72, 0x5f, 0x06, 0xd2, 0x57, 0x71, 0xca, 0x5f, 0x92, 0xb9, 0x2f, 0x23, 0xdf, 0x88, - 0x44, 0xa6, 0x42, 0x0b, 0x84, 0xf2, 0x22, 0x90, 0xc4, 0x38, 0x48, 0x32, 0xdf, 0x6b, 0xc8, 0xc8, - 0x97, 0xa9, 0x10, 0x0f, 0x1b, 0x8f, 0x77, 0x0b, 0xad, 0x0b, 0xae, 0xc7, 0x99, 0x51, 0xf5, 0x5e, - 0x43, 0xf6, 0xc8, 0x95, 0x64, 0xf7, 0x9c, 0xf2, 0xa7, 0x05, 0x57, 0x1a, 0xb5, 0xa0, 0x9a, 0x0a, - 0xa1, 0x67, 0x31, 0x53, 0x71, 0xd3, 0x6e, 0xdb, 0x9d, 0x3a, 0xad, 0x64, 0xc2, 0x25, 0x53, 0x31, - 0xda, 0x87, 0x6a, 0xf2, 0x3d, 0xd0, 0x2c, 0x99, 0x66, 0x21, 0x78, 0x77, 0xe0, 0xfd, 0x45, 0x1e, - 0x6b, 0xa6, 0x17, 0x8a, 0x72, 0x25, 0x45, 0xa2, 0x38, 0x3a, 0x06, 0x47, 0x19, 0xc5, 0xd0, 0x1b, - 0x01, 0x26, 0xbf, 0x43, 0x93, 0xcd, 0xcc, 0xa9, 0x98, 0x73, 0x9a, 0xbb, 0xbd, 0x29, 0xec, 0x14, - 0x61, 0xc5, 0xf3, 0x0f, 0x6f, 0x17, 0x1c, 0x03, 0xc8, 0x78, 0xe5, 0x4e, 0x9d, 0xe6, 0x15, 0x3a, - 0x80, 0x7f, 0x66, 0x6d, 0x93, 0xb3, 0x16, 0x6c, 0x93, 0xfc, 0x08, 0x11, 0xb9, 0xce, 0x1e, 0x74, - 0xd3, 0x3d, 0x3c, 0x01, 0x28, 0x3e, 0x43, 0x35, 0xf8, 0x3f, 0x08, 0x6f, 0xba, 0xc3, 0x41, 0xdf, - 0xb5, 0x90, 0x03, 0xa5, 0xd1, 0x95, 0x6b, 0xa3, 0x2d, 0xa8, 0x86, 0xa3, 0xc9, 0xec, 0x7c, 0x34, - 0x0d, 0xfb, 0x6e, 0x09, 0xd5, 0xa1, 0x32, 0x08, 0x27, 0x67, 0x34, 0xec, 0x0e, 0xdd, 0x72, 0xaf, - 0xf9, 0xbe, 0xc2, 0xf6, 0x72, 0x85, 0xed, 0xcf, 0x15, 0xb6, 0xdf, 0xd6, 0xd8, 0x5a, 0xae, 0xb1, - 0xf5, 0xb1, 0xc6, 0x56, 0xe4, 0x98, 0x7b, 0x1f, 0x7d, 0x05, 0x00, 0x00, 0xff, 0xff, 0x1a, 0x53, - 0xb4, 0x86, 0xb7, 0x01, 0x00, 0x00, + // 353 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0x4f, 0x4b, 0xe3, 0x40, + 0x18, 0xc6, 0x33, 0xfd, 0x93, 0x6e, 0xdf, 0xfe, 0xd9, 0x30, 0x94, 0x12, 0x76, 0x97, 0x10, 0x02, + 0x0b, 0x61, 0x0f, 0x09, 0x64, 0xc1, 0xa3, 0xd0, 0x5a, 0x95, 0x62, 0x49, 0x65, 0xda, 0x7a, 0x12, + 0x24, 0xb1, 0x53, 0xe3, 0xa1, 0x99, 0x31, 0x93, 0x52, 0x3d, 0xfb, 0x05, 0xfc, 0x58, 0x1e, 0x7b, + 0xf4, 0x28, 0xed, 0x17, 0x91, 0x4c, 0x52, 0x7b, 0xd0, 0x5b, 0x9e, 0x27, 0xbf, 0x79, 0xe6, 0x7d, + 0xde, 0x01, 0x53, 0x44, 0x41, 0x42, 0x5d, 0xee, 0x71, 0x57, 0x44, 0x09, 0x7d, 0x8c, 0xe7, 0x2e, + 0x0f, 0x5d, 0x69, 0x3a, 0x3c, 0x61, 0x29, 0xc3, 0xb8, 0x10, 0x1e, 0x77, 0x24, 0xe1, 0xc4, 0xf3, + 0x5f, 0x6d, 0x1e, 0xba, 0x3c, 0x61, 0x6c, 0x91, 0x33, 0xd6, 0x33, 0x82, 0xdf, 0xe7, 0x34, 0x9d, + 0x64, 0xa4, 0xe8, 0x3f, 0xf9, 0xc1, 0x92, 0x0a, 0x1e, 0xdc, 0x52, 0x42, 0x1f, 0x56, 0x54, 0xa4, + 0xb8, 0x0b, 0x6a, 0x44, 0xef, 0xef, 0xa2, 0x54, 0x47, 0x26, 0xb2, 0x2b, 0xa4, 0x50, 0xf8, 0x0f, + 0xd4, 0xe3, 0x3d, 0xab, 0x97, 0x4c, 0x64, 0x37, 0xc9, 0xc1, 0xc0, 0x3a, 0xd4, 0x16, 0x09, 0x5b, + 0x12, 0xb6, 0xd6, 0xcb, 0x26, 0xb2, 0x5b, 0x64, 0x2f, 0x71, 0x07, 0xaa, 0x29, 0xcb, 0xfc, 0x8a, + 0xf4, 0x73, 0x61, 0x5d, 0x83, 0xf5, 0xdd, 0x10, 0x93, 0x34, 0x48, 0x57, 0x82, 0x50, 0xc1, 0x59, + 0x2c, 0x28, 0x3e, 0x02, 0x55, 0x48, 0x47, 0xce, 0xd2, 0xf6, 0x0c, 0xe7, 0x6b, 0x41, 0x27, 0x3f, + 0x73, 0xc2, 0xe6, 0x94, 0x14, 0xb4, 0x35, 0x83, 0xce, 0xa1, 0x17, 0x5b, 0x7f, 0xe6, 0x75, 0x41, + 0x95, 0x01, 0x59, 0x5e, 0xd9, 0x6e, 0x92, 0x42, 0xe1, 0xbf, 0x50, 0x95, 0x2b, 0x92, 0xbd, 0x1a, + 0xde, 0x4f, 0xa7, 0x58, 0x58, 0xe8, 0x5c, 0x66, 0x1f, 0x24, 0xff, 0xfb, 0xef, 0x18, 0xe0, 0x70, + 0x19, 0x6e, 0x40, 0x6d, 0xe8, 0x5f, 0xf5, 0x46, 0xc3, 0x81, 0xa6, 0x60, 0x15, 0x4a, 0xe3, 0x0b, + 0x0d, 0xe1, 0x16, 0xd4, 0xfd, 0xf1, 0xf4, 0xe6, 0x6c, 0x3c, 0xf3, 0x07, 0x5a, 0x09, 0x37, 0xe1, + 0xc7, 0xd0, 0x9f, 0x9e, 0x12, 0xbf, 0x37, 0xd2, 0xca, 0x7d, 0xfd, 0x75, 0x6b, 0xa0, 0xcd, 0xd6, + 0x40, 0xef, 0x5b, 0x03, 0xbd, 0xec, 0x0c, 0x65, 0xb3, 0x33, 0x94, 0xb7, 0x9d, 0xa1, 0x84, 0xaa, + 0x7c, 0x9b, 0xff, 0x1f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x43, 0xb4, 0x09, 0x76, 0xe3, 0x01, 0x00, + 0x00, } func (m *GetSharesByNamespaceRequest) Marshal() (dAtA []byte, err error) { @@ -256,6 +274,16 @@ func (m *GetSharesByNamespaceRequest) MarshalToSizedBuffer(dAtA []byte) (int, er _ = i var l int _ = l + if m.ToRow != 0 { + i = encodeVarintShare(dAtA, i, uint64(m.ToRow)) + i-- + dAtA[i] = 0x20 + } + if m.FromRow != 0 { + i = encodeVarintShare(dAtA, i, uint64(m.FromRow)) + i-- + dAtA[i] = 0x18 + } if len(m.Namespace) > 0 { i -= len(m.Namespace) copy(dAtA[i:], m.Namespace) @@ -263,12 +291,10 @@ func (m *GetSharesByNamespaceRequest) MarshalToSizedBuffer(dAtA []byte) (int, er i-- dAtA[i] = 0x12 } - if len(m.RootHash) > 0 { - i -= len(m.RootHash) - copy(dAtA[i:], m.RootHash) - i = encodeVarintShare(dAtA, i, uint64(len(m.RootHash))) + if m.Height != 0 { + i = encodeVarintShare(dAtA, i, uint64(m.Height)) i-- - dAtA[i] = 0xa + dAtA[i] = 0x8 } return len(dAtA) - i, nil } @@ -362,14 +388,19 @@ func (m *GetSharesByNamespaceRequest) Size() (n int) { } var l int _ = l - l = len(m.RootHash) - if l > 0 { - n += 1 + l + sovShare(uint64(l)) + if m.Height != 0 { + n += 1 + sovShare(uint64(m.Height)) } l = len(m.Namespace) if l > 0 { n += 1 + l + sovShare(uint64(l)) } + if m.FromRow != 0 { + n += 1 + sovShare(uint64(m.FromRow)) + } + if m.ToRow != 0 { + n += 1 + sovShare(uint64(m.ToRow)) + } return n } @@ -440,10 +471,10 @@ func (m *GetSharesByNamespaceRequest) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field RootHash", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Height", wireType) } - var byteLen int + m.Height = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowShare @@ -453,26 +484,11 @@ func (m *GetSharesByNamespaceRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + m.Height |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { - return ErrInvalidLengthShare - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthShare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.RootHash = append(m.RootHash[:0], dAtA[iNdEx:postIndex]...) - if m.RootHash == nil { - m.RootHash = []byte{} - } - iNdEx = postIndex case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) @@ -507,6 +523,44 @@ func (m *GetSharesByNamespaceRequest) Unmarshal(dAtA []byte) error { m.Namespace = []byte{} } iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FromRow", wireType) + } + m.FromRow = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShare + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FromRow |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ToRow", wireType) + } + m.ToRow = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShare + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ToRow |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipShare(dAtA[iNdEx:]) diff --git a/share/p2p/shrexnd/pb/share.proto b/share/p2p/shrexnd/pb/share.proto index a5bdbfa071..a1c9c17a6d 100644 --- a/share/p2p/shrexnd/pb/share.proto +++ b/share/p2p/shrexnd/pb/share.proto @@ -4,8 +4,10 @@ package share.p2p.shrex.nd; import "pb/proof.proto"; message GetSharesByNamespaceRequest{ - bytes root_hash = 1; + uint64 height = 1; bytes namespace = 2; + uint32 fromRow = 3; + uint32 toRow = 4; } message GetSharesByNamespaceStatusResponse{ diff --git a/share/p2p/shrexnd/server.go b/share/p2p/shrexnd/server.go index 33e61ff472..a7553efcf9 100644 --- a/share/p2p/shrexnd/server.go +++ b/share/p2p/shrexnd/server.go @@ -2,9 +2,9 @@ package shrexnd import ( "context" - "crypto/sha256" "errors" "fmt" + "github.com/celestiaorg/celestia-node/share/ipld" "time" "github.com/libp2p/go-libp2p/core/host" @@ -15,10 +15,11 @@ import ( "github.com/celestiaorg/go-libp2p-messenger/serde" nmt_pb "github.com/celestiaorg/nmt/pb" + "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/p2p" pb "github.com/celestiaorg/celestia-node/share/p2p/shrexnd/pb" + "github.com/celestiaorg/celestia-node/share/store" ) // Server implements server side of shrex/nd protocol to serve namespaced share to remote @@ -30,7 +31,7 @@ type Server struct { protocolID protocol.ID handler network.StreamHandler - store *eds.Store + store *store.Store params *Parameters middleware *p2p.Middleware @@ -38,7 +39,7 @@ type Server struct { } // NewServer creates new Server -func NewServer(params *Parameters, host host.Host, store *eds.Store) (*Server, error) { +func NewServer(params *Parameters, host host.Host, store *store.Store) (*Server, error) { if err := params.Validate(); err != nil { return nil, fmt.Errorf("shrex-nd: server creation failed: %w", err) } @@ -108,13 +109,15 @@ func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stre return err } - logger = logger.With("namespace", share.Namespace(req.Namespace).String(), - "hash", share.DataHash(req.RootHash).String()) + logger = logger.With( + "namespace", share.Namespace(req.Namespace).String(), + "height", req.Height, + ) ctx, cancel := context.WithTimeout(ctx, srv.params.HandleRequestTimeout) defer cancel() - shares, status, err := srv.getNamespaceData(ctx, req.RootHash, req.Namespace) + shares, status, err := srv.getNamespaceData(ctx, req.Height, req.Namespace, int(req.FromRow), int(req.ToRow)) if err != nil { // server should respond with status regardless if there was an error getting data sendErr := srv.respondStatus(ctx, logger, stream, status) @@ -172,21 +175,38 @@ func (srv *Server) readRequest( } func (srv *Server) getNamespaceData(ctx context.Context, - hash share.DataHash, namespace share.Namespace) (share.NamespacedShares, pb.StatusCode, error) { - dah, err := srv.store.GetDAH(ctx, hash) + height uint64, + namespace share.Namespace, + fromRow, toRow int, +) (share.NamespacedShares, pb.StatusCode, error) { + file, err := srv.store.GetByHeight(ctx, height) if err != nil { - if errors.Is(err, eds.ErrNotFound) { + if errors.Is(err, store.ErrNotFound) { return nil, pb.StatusCode_NOT_FOUND, nil } return nil, pb.StatusCode_INTERNAL, fmt.Errorf("retrieving DAH: %w", err) } + defer utils.CloseAndLog(log, "file", file) - shares, err := eds.RetrieveNamespaceFromStore(ctx, srv.store, dah, namespace) - if err != nil { - return nil, pb.StatusCode_INTERNAL, fmt.Errorf("retrieving shares: %w", err) + if toRow > file.Size()/2 { + // TODO(@Walldiss): needs refactoring for better handling + return nil, pb.StatusCode_NOT_FOUND, fmt.Errorf("toRow: (%d) exceeds file ods size: (%d)", toRow, file.Size()) + } + + namespacedRows := make(share.NamespacedShares, 0, toRow-fromRow+1) + for rowIdx := fromRow; rowIdx < toRow; rowIdx++ { + data, err := file.Data(ctx, namespace, rowIdx) + if err != nil { + if errors.Is(err, ipld.ErrNamespaceOutsideRange) { + // TODO(@Walldiss): needs refactoring for better handling + return nil, pb.StatusCode_NOT_FOUND, fmt.Errorf("namespace outside range: %w for row %d", err, rowIdx) + } + return nil, pb.StatusCode_INTERNAL, fmt.Errorf("retrieving data: %w", err) + } + namespacedRows = append(namespacedRows, data) } - return shares, pb.StatusCode_OK, nil + return namespacedRows, pb.StatusCode_OK, nil } func (srv *Server) respondStatus( @@ -244,11 +264,8 @@ func (srv *Server) observeStatus(ctx context.Context, status pb.StatusCode) { // validateRequest checks correctness of the request func validateRequest(req pb.GetSharesByNamespaceRequest) error { - if err := share.Namespace(req.Namespace).ValidateForData(); err != nil { - return err + if req.ToRow < req.FromRow { + return fmt.Errorf("invalid request: ToRow must be greater than FromRow") } - if len(req.RootHash) != sha256.Size { - return fmt.Errorf("incorrect root hash length: %v", len(req.RootHash)) - } - return nil + return share.Namespace(req.Namespace).ValidateForData() } diff --git a/share/p2p/shrexsub/pubsub.go b/share/p2p/shrexsub/pubsub.go index ed713b4614..64d7239c63 100644 --- a/share/p2p/shrexsub/pubsub.go +++ b/share/p2p/shrexsub/pubsub.go @@ -17,7 +17,7 @@ var log = logging.Logger("shrex-sub") // pubsubTopic hardcodes the name of the EDS floodsub topic with the provided networkID. func pubsubTopicID(networkID string) string { - return fmt.Sprintf("%s/eds-sub/v0.1.0", networkID) + return fmt.Sprintf("%s/eds-sub/v0.2.0", networkID) } // ValidatorFn is an injectable func and governs EDS notification msg validity. diff --git a/share/share.go b/share/share.go index 4079028d82..cddbedcdb3 100644 --- a/share/share.go +++ b/share/share.go @@ -2,10 +2,13 @@ package share import ( "bytes" + "crypto/sha256" "encoding/hex" "fmt" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" ) var ( @@ -40,6 +43,31 @@ func GetData(s Share) []byte { return s[NamespaceSize:] } +// ShareWithProof contains data with corresponding Merkle Proof +type ShareWithProof struct { //nolint: revive + // Share is a full data including namespace + Share + // Proof is a Merkle Proof of current share + Proof *nmt.Proof + // Axis is a type of axis against which the share proof is computed + Axis rsmt2d.Axis +} + +// Validate validates inclusion of the share under the given root CID. +func (s *ShareWithProof) Validate(rootHash []byte, x, y, edsSize int) bool { + isParity := x >= edsSize/2 || y >= edsSize/2 + namespace := ParitySharesNamespace + if !isParity { + namespace = GetNamespace(s.Share) + } + return s.Proof.VerifyInclusion( + sha256.New(), // TODO(@Wondertan): This should be defined somewhere globally + namespace.ToNMT(), + [][]byte{s.Share}, + rootHash, + ) +} + // DataHash is a representation of the Root hash. type DataHash []byte diff --git a/share/shwap/data.go b/share/shwap/data.go new file mode 100644 index 0000000000..5cd96a780c --- /dev/null +++ b/share/shwap/data.go @@ -0,0 +1,169 @@ +package shwap + +import ( + "context" + "fmt" + + blocks "github.com/ipfs/go-block-format" + + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" + nmtpb "github.com/celestiaorg/nmt/pb" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/ipld" + shwappb "github.com/celestiaorg/celestia-node/share/shwap/pb" +) + +type Data struct { + DataID + + DataShares []share.Share + DataProof nmt.Proof +} + +// NewData constructs a new Data. +func NewData(id DataID, shares []share.Share, proof nmt.Proof) *Data { + return &Data{ + DataID: id, + DataShares: shares, + DataProof: proof, + } +} + +// NewDataFromEDS samples the EDS and constructs Data for each row with the given namespace. +func NewDataFromEDS( + square *rsmt2d.ExtendedDataSquare, + height uint64, + namespace share.Namespace, +) ([]*Data, error) { + root, err := share.NewRoot(square) + if err != nil { + return nil, fmt.Errorf("while computing root: %w", err) + } + + var datas []*Data //nolint:prealloc// we don't know how many rows with needed namespace there are + for rowIdx, rowRoot := range root.RowRoots { + if namespace.IsOutsideRange(rowRoot, rowRoot) { + continue + } + + id, err := NewDataID(height, uint16(rowIdx), namespace, root) + if err != nil { + return nil, err + } + + shrs := square.Row(uint(rowIdx)) + // TDOD(@Wondertan): This will likely be removed + nd, proof, err := ndFromShares(shrs, namespace, rowIdx) + if err != nil { + return nil, err + } + + datas = append(datas, NewData(id, nd, proof)) + } + + return datas, nil +} + +func ndFromShares(shrs []share.Share, namespace share.Namespace, axisIdx int) ([]share.Share, nmt.Proof, error) { + bserv := ipld.NewMemBlockservice() + batchAdder := ipld.NewNmtNodeAdder(context.TODO(), bserv, ipld.MaxSizeBatchOption(len(shrs))) + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(len(shrs)/2), uint(axisIdx), + nmt.NodeVisitor(batchAdder.Visit)) + for _, shr := range shrs { + err := tree.Push(shr) + if err != nil { + return nil, nmt.Proof{}, err + } + } + + root, err := tree.Root() + if err != nil { + return nil, nmt.Proof{}, err + } + + err = batchAdder.Commit() + if err != nil { + return nil, nmt.Proof{}, err + } + + row, proof, err := ipld.GetSharesByNamespace(context.TODO(), bserv, root, namespace, len(shrs)) + if err != nil { + return nil, nmt.Proof{}, err + } + return row, *proof, nil +} + +// DataFromBlock converts blocks.Block into Data. +func DataFromBlock(blk blocks.Block) (*Data, error) { + if err := validateCID(blk.Cid()); err != nil { + return nil, err + } + return DataFromBinary(blk.RawData()) +} + +// IPLDBlock converts Data to an IPLD block for Bitswap compatibility. +func (s *Data) IPLDBlock() (blocks.Block, error) { + data, err := s.MarshalBinary() + if err != nil { + return nil, err + } + + return blocks.NewBlockWithCid(data, s.Cid()) +} + +// MarshalBinary marshals Data to binary. +func (s *Data) MarshalBinary() ([]byte, error) { + did := s.DataID.MarshalBinary() + proof := &nmtpb.Proof{} + proof.Nodes = s.DataProof.Nodes() + proof.End = int64(s.DataProof.End()) + proof.Start = int64(s.DataProof.Start()) + proof.IsMaxNamespaceIgnored = s.DataProof.IsMaxNamespaceIDIgnored() + proof.LeafHash = s.DataProof.LeafHash() + + return (&shwappb.Data{ + DataId: did, + DataShares: s.DataShares, + DataProof: proof, + }).Marshal() +} + +// DataFromBinary unmarshal Data from binary. +func DataFromBinary(data []byte) (*Data, error) { + proto := &shwappb.Data{} + if err := proto.Unmarshal(data); err != nil { + return nil, err + } + + did, err := DataIDFromBinary(proto.DataId) + if err != nil { + return nil, err + } + return NewData(did, proto.DataShares, nmt.ProtoToProof(*proto.DataProof)), nil +} + +// Verify validates Data's fields and verifies Data inclusion. +func (s *Data) Verify(root *share.Root) error { + if err := s.DataID.Verify(root); err != nil { + return err + } + + if len(s.DataShares) == 0 && s.DataProof.IsEmptyProof() { + return fmt.Errorf("empty Data") + } + + shrs := make([][]byte, 0, len(s.DataShares)) + for _, shr := range s.DataShares { + shrs = append(shrs, append(share.GetNamespace(shr), shr...)) + } + + rowRoot := root.RowRoots[s.RowIndex] + if !s.DataProof.VerifyNamespace(hashFn(), s.Namespace().ToNMT(), shrs, rowRoot) { + return fmt.Errorf("invalid DataProof") + } + + return nil +} diff --git a/share/shwap/data_hasher.go b/share/shwap/data_hasher.go new file mode 100644 index 0000000000..932172b523 --- /dev/null +++ b/share/shwap/data_hasher.go @@ -0,0 +1,60 @@ +package shwap + +import ( + "crypto/sha256" + "fmt" +) + +// DataHasher implements hash.Hash interface for Data. +type DataHasher struct { + data []byte +} + +// Write expects a marshaled Data to validate. +func (h *DataHasher) Write(data []byte) (int, error) { + d, err := DataFromBinary(data) + if err != nil { + err = fmt.Errorf("unmarshaling Data: %w", err) + log.Error(err) + return 0, err + } + + root, err := getRoot(d.DataID) + if err != nil { + err = fmt.Errorf("getting root: %w", err) + return 0, err + } + + if err := d.Verify(root); err != nil { + err = fmt.Errorf("verifying Data: %w", err) + log.Error(err) + return 0, err + } + + h.data = data + return len(data), nil +} + +// Sum returns the "multihash" of the DataID. +func (h *DataHasher) Sum([]byte) []byte { + if h.data == nil { + return nil + } + const pbOffset = 2 + return h.data[pbOffset : DataIDSize+pbOffset] +} + +// Reset resets the Hash to its initial state. +func (h *DataHasher) Reset() { + h.data = nil +} + +// Size returns the number of bytes Sum will return. +func (h *DataHasher) Size() int { + return DataIDSize +} + +// BlockSize returns the hash's underlying block size. +func (h *DataHasher) BlockSize() int { + return sha256.BlockSize +} diff --git a/share/shwap/data_hasher_test.go b/share/shwap/data_hasher_test.go new file mode 100644 index 0000000000..9dfb0ce1c6 --- /dev/null +++ b/share/shwap/data_hasher_test.go @@ -0,0 +1,43 @@ +package shwap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" +) + +func TestDataHasher(t *testing.T) { + hasher := &DataHasher{} + + _, err := hasher.Write([]byte("hello")) + assert.Error(t, err) + + size := 8 + namespace := sharetest.RandV0Namespace() + square, root := edstest.RandEDSWithNamespace(t, namespace, size*size, size) + + datas, err := NewDataFromEDS(square, 1, namespace) + require.NoError(t, err) + data := datas[0] + + globalRootsCache.Store(data.DataID, root) + + dat, err := data.MarshalBinary() + require.NoError(t, err) + + n, err := hasher.Write(dat) + require.NoError(t, err) + assert.EqualValues(t, len(dat), n) + + digest := hasher.Sum(nil) + id := data.DataID.MarshalBinary() + assert.EqualValues(t, id, digest) + + hasher.Reset() + digest = hasher.Sum(nil) + assert.NotEqualValues(t, digest, id) +} diff --git a/share/shwap/data_id.go b/share/shwap/data_id.go new file mode 100644 index 0000000000..7241cddbc9 --- /dev/null +++ b/share/shwap/data_id.go @@ -0,0 +1,139 @@ +package shwap + +import ( + "context" + "fmt" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/store/file" +) + +// DataIDSize is the size of the DataID in bytes. +const DataIDSize = RowIDSize + share.NamespaceSize + +// DataID is an unique identifier of a namespaced Data inside EDS Row. +type DataID struct { + RowID + + // DataNamespace is the namespace of the data + // It's string formatted to keep DataID comparable + DataNamespace string +} + +// NewDataID constructs a new DataID. +func NewDataID(height uint64, rowIdx uint16, namespace share.Namespace, root *share.Root) (DataID, error) { + did := DataID{ + RowID: RowID{ + EdsID: EdsID{ + Height: height, + }, + RowIndex: rowIdx, + }, + DataNamespace: string(namespace), + } + + // Store the root in the cache for verification later + globalRootsCache.Store(did, root) + return did, did.Verify(root) +} + +// DataIDFromCID coverts CID to DataID. +func DataIDFromCID(cid cid.Cid) (id DataID, err error) { + if err = validateCID(cid); err != nil { + return id, err + } + + id, err = DataIDFromBinary(cid.Hash()[mhPrefixSize:]) + if err != nil { + return id, fmt.Errorf("unmarhalling DataID: %w", err) + } + + return id, nil +} + +// Namespace returns the namespace of the DataID. +func (s DataID) Namespace() share.Namespace { + return share.Namespace(s.DataNamespace) +} + +// Cid returns DataID encoded as CID. +func (s DataID) Cid() cid.Cid { + // avoid using proto serialization for CID as it's not deterministic + data := s.MarshalBinary() + + buf, err := mh.Encode(data, dataMultihashCode) + if err != nil { + panic(fmt.Errorf("encoding DataID as CID: %w", err)) + } + + return cid.NewCidV1(dataCodec, buf) +} + +// MarshalBinary encodes DataID into binary form. +// NOTE: Proto is avoided because +// * Its size is not deterministic which is required for IPLD. +// * No support for uint16 +func (s DataID) MarshalBinary() []byte { + data := make([]byte, 0, DataIDSize) + return s.appendTo(data) +} + +// DataIDFromBinary decodes DataID from binary form. +func DataIDFromBinary(data []byte) (DataID, error) { + var did DataID + if len(data) != DataIDSize { + return did, fmt.Errorf("invalid DataID data length: %d != %d", len(data), DataIDSize) + } + rid, err := RowIDFromBinary(data[:RowIDSize]) + if err != nil { + return did, fmt.Errorf("while unmarhaling RowID: %w", err) + } + did.RowID = rid + ns := share.Namespace(data[RowIDSize:]) + if err = ns.ValidateForData(); err != nil { + return did, fmt.Errorf("validating DataNamespace: %w", err) + } + did.DataNamespace = string(ns) + return did, err +} + +// Verify verifies DataID fields. +func (s DataID) Verify(root *share.Root) error { + if err := s.RowID.Verify(root); err != nil { + return fmt.Errorf("validating RowID: %w", err) + } + if err := s.Namespace().ValidateForData(); err != nil { + return fmt.Errorf("validating DataNamespace: %w", err) + } + + return nil +} + +// BlockFromFile returns the IPLD block of the DataID from the given file. +func (s DataID) BlockFromFile(ctx context.Context, f file.EdsFile) (blocks.Block, error) { + data, err := f.Data(ctx, s.Namespace(), int(s.RowIndex)) + if err != nil { + return nil, fmt.Errorf("while getting Data: %w", err) + } + + d := NewData(s, data.Shares, *data.Proof) + blk, err := d.IPLDBlock() + if err != nil { + return nil, fmt.Errorf("while coverting Data to IPLD block: %w", err) + } + return blk, nil +} + +// Release releases the verifier of the DataID. +func (s DataID) Release() { + globalRootsCache.Delete(s) +} + +func (s DataID) appendTo(data []byte) []byte { + data = s.RowID.appendTo(data) + return append(data, s.DataNamespace...) +} diff --git a/share/shwap/data_id_test.go b/share/shwap/data_id_test.go new file mode 100644 index 0000000000..9963a4ecba --- /dev/null +++ b/share/shwap/data_id_test.go @@ -0,0 +1,32 @@ +package shwap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" +) + +func TestDataID(t *testing.T) { + ns := sharetest.RandV0Namespace() + _, root := edstest.RandEDSWithNamespace(t, ns, 8, 4) + + id, err := NewDataID(1, 1, ns, root) + require.NoError(t, err) + + cid := id.Cid() + assert.EqualValues(t, dataCodec, cid.Prefix().Codec) + assert.EqualValues(t, dataMultihashCode, cid.Prefix().MhType) + assert.EqualValues(t, DataIDSize, cid.Prefix().MhLength) + + data := id.MarshalBinary() + sidOut, err := DataIDFromBinary(data) + require.NoError(t, err) + assert.EqualValues(t, id, sidOut) + + err = sidOut.Verify(root) + require.NoError(t, err) +} diff --git a/share/shwap/data_test.go b/share/shwap/data_test.go new file mode 100644 index 0000000000..c41373aedc --- /dev/null +++ b/share/shwap/data_test.go @@ -0,0 +1,34 @@ +package shwap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" +) + +func TestData(t *testing.T) { + namespace := sharetest.RandV0Namespace() + square, root := edstest.RandEDSWithNamespace(t, namespace, 16, 8) + + nds, err := NewDataFromEDS(square, 1, namespace) + require.NoError(t, err) + nd := nds[0] + + data, err := nd.MarshalBinary() + require.NoError(t, err) + + blk, err := nd.IPLDBlock() + require.NoError(t, err) + assert.EqualValues(t, blk.Cid(), nd.Cid()) + + dataOut, err := DataFromBinary(data) + require.NoError(t, err) + assert.EqualValues(t, nd, dataOut) + + err = dataOut.Verify(root) + require.NoError(t, err) +} diff --git a/share/shwap/eds_id.go b/share/shwap/eds_id.go new file mode 100644 index 0000000000..83cb07ecad --- /dev/null +++ b/share/shwap/eds_id.go @@ -0,0 +1,61 @@ +package shwap + +import ( + "encoding/binary" + "fmt" + + "github.com/celestiaorg/celestia-node/share" +) + +// EdsIDSize is the size of the EdsID in bytes +const EdsIDSize = 8 + +// EdsID is an unique identifier of a Row. +type EdsID struct { + // Height of the block. + // Needed to identify block's data square in the whole chain + Height uint64 +} + +// NewEdsID constructs a new EdsID. +func NewEdsID(height uint64, root *share.Root) (EdsID, error) { + rid := EdsID{ + Height: height, + } + return rid, rid.Verify(root) +} + +// MarshalBinary encodes EdsID into binary form. +func (eid EdsID) MarshalBinary() []byte { + data := make([]byte, 0, EdsIDSize) + return eid.appendTo(data) +} + +// EdsIDFromBinary decodes EdsID from binary form. +func EdsIDFromBinary(data []byte) (rid EdsID, err error) { + if len(data) != EdsIDSize { + return rid, fmt.Errorf("invalid EdsID data length: %d != %d", len(data), EdsIDSize) + } + rid.Height = binary.BigEndian.Uint64(data) + return rid, nil +} + +// Verify verifies EdsID fields. +func (eid EdsID) Verify(root *share.Root) error { + if root == nil { + return fmt.Errorf("nil Root") + } + if eid.Height == 0 { + return fmt.Errorf("zero Height") + } + + return nil +} + +func (eid EdsID) GetHeight() uint64 { + return eid.Height +} + +func (eid EdsID) appendTo(data []byte) []byte { + return binary.BigEndian.AppendUint64(data, eid.Height) +} diff --git a/share/shwap/eds_id_test.go b/share/shwap/eds_id_test.go new file mode 100644 index 0000000000..9f0a3190fb --- /dev/null +++ b/share/shwap/eds_id_test.go @@ -0,0 +1,28 @@ +package shwap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/testing/edstest" +) + +func TestEdsID(t *testing.T) { + square := edstest.RandEDS(t, 2) + root, err := share.NewRoot(square) + require.NoError(t, err) + + id, err := NewEdsID(2, root) + require.NoError(t, err) + + data := id.MarshalBinary() + idOut, err := EdsIDFromBinary(data) + require.NoError(t, err) + assert.EqualValues(t, id, idOut) + + err = idOut.Verify(root) + require.NoError(t, err) +} diff --git a/share/shwap/getter/getter.go b/share/shwap/getter/getter.go new file mode 100644 index 0000000000..7e865ad27e --- /dev/null +++ b/share/shwap/getter/getter.go @@ -0,0 +1,238 @@ +package shwap_getter + +import ( + "context" + "fmt" + + "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/exchange" + block "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/shwap" +) + +type Getter struct { + // TODO(@walldiss): why not blockservice? + fetch exchange.SessionExchange + bstore blockstore.Blockstore +} + +func NewGetter(fetch exchange.SessionExchange, bstore blockstore.Blockstore) *Getter { + return &Getter{fetch: fetch, bstore: bstore} +} + +func (g *Getter) GetShare(ctx context.Context, header *header.ExtendedHeader, row, col int) (share.Share, error) { + shrIdx := row*len(header.DAH.RowRoots) + col + shrs, err := g.GetShares(ctx, header, shrIdx) + if err != nil { + return nil, fmt.Errorf("getting shares: %w", err) + } + + if len(shrs) != 1 { + return nil, fmt.Errorf("expected 1 share, got %d", len(shrs)) + } + + return shrs[0], nil +} + +// TODO: Make GetSamples so it provides proofs to users. +// GetShares fetches in the Block/EDS by their indexes. +// Automatically caches them on the Blockstore. +// Guarantee that the returned shares are in the same order as shrIdxs. +func (g *Getter) GetShares(ctx context.Context, hdr *header.ExtendedHeader, smplIdxs ...int) ([]share.Share, error) { + if len(smplIdxs) == 0 { + return nil, nil + } + + if hdr.DAH.Equals(share.EmptyRoot()) { + shares := make([]share.Share, len(smplIdxs)) + for _, idx := range smplIdxs { + x, y := uint(smplIdxs[idx]/len(hdr.DAH.RowRoots)), uint(smplIdxs[idx]%len(hdr.DAH.RowRoots)) + shares[idx] = share.EmptyExtendedDataSquare().GetCell(x, y) + } + return shares, nil + } + + cids := make([]cid.Cid, len(smplIdxs)) + for i, shrIdx := range smplIdxs { + sid, err := shwap.NewSampleID(hdr.Height(), shrIdx, hdr.DAH) + if err != nil { + return nil, err + } + defer sid.Release() + cids[i] = sid.Cid() + } + + blks, err := g.getBlocks(ctx, cids) + if err != nil { + return nil, fmt.Errorf("getting blocks: %w", err) + } + + // ensure we persist samples/blks and make them available for Bitswap + err = g.bstore.PutMany(ctx, blks) + if err != nil { + return nil, fmt.Errorf("storing shares: %w", err) + } + // tell bitswap that we stored the blks and can serve them now + err = g.fetch.NotifyNewBlocks(ctx, blks...) + if err != nil { + return nil, fmt.Errorf("notifying new shares: %w", err) + } + + // ensure we return shares in the requested order + shares := make(map[int]share.Share, len(blks)) + for _, blk := range blks { + sample, err := shwap.SampleFromBlock(blk) + if err != nil { + return nil, fmt.Errorf("getting sample from block: %w", err) + } + shrIdx := int(sample.SampleID.RowIndex)*len(hdr.DAH.RowRoots) + int(sample.SampleID.ShareIndex) + shares[shrIdx] = sample.SampleShare + } + + ordered := make([]share.Share, len(shares)) + for i, shrIdx := range smplIdxs { + sh, ok := shares[shrIdx] + if !ok { + return nil, fmt.Errorf("missing share for index %d", shrIdx) + } + ordered[i] = sh + } + + return ordered, nil +} + +// GetEDS +// TODO(@Wondertan): Consider requesting randomized rows instead of ODS only +func (g *Getter) GetEDS(ctx context.Context, hdr *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { + if hdr.DAH.Equals(share.EmptyRoot()) { + return share.EmptyExtendedDataSquare(), nil + } + + sqrLn := len(hdr.DAH.RowRoots) + cids := make([]cid.Cid, sqrLn/2) + for i := 0; i < sqrLn/2; i++ { + rid, err := shwap.NewRowID(hdr.Height(), uint16(i), hdr.DAH) + if err != nil { + return nil, err + } + defer rid.Release() + cids[i] = rid.Cid() + } + + blks, err := g.getBlocks(ctx, cids) + if err != nil { + return nil, fmt.Errorf("getting blocks: %w", err) + + } + + rows := make([]*shwap.Row, len(blks)) + for _, blk := range blks { + row, err := shwap.RowFromBlock(blk) + if err != nil { + return nil, fmt.Errorf("getting row from block: %w", err) + } + rows[row.RowIndex] = row + } + + shrs := make([]share.Share, 0, sqrLn*sqrLn) + for _, row := range rows { + shrs = append(shrs, row.RowShares...) + } + + square, err := rsmt2d.ComputeExtendedDataSquare( + shrs, + share.DefaultRSMT2DCodec(), + wrapper.NewConstructor(uint64(sqrLn/2)), + ) + if err != nil { + return nil, fmt.Errorf("computing EDS: %w", err) + } + + // and try to repair + err = square.Repair(hdr.DAH.RowRoots, hdr.DAH.ColumnRoots) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("repairing EDS: %w", err) + } + + return square, nil +} + +func (g *Getter) GetSharesByNamespace( + ctx context.Context, + hdr *header.ExtendedHeader, + ns share.Namespace, +) (share.NamespacedShares, error) { + if err := ns.ValidateForData(); err != nil { + return nil, err + } + + from, to := share.RowRangeForNamespace(hdr.DAH, ns) + if from == to { + return share.NamespacedShares{}, nil + } + + cids := make([]cid.Cid, 0, to-from) + for rowIdx := from; rowIdx < to; rowIdx++ { + did, err := shwap.NewDataID(hdr.Height(), uint16(rowIdx), ns, hdr.DAH) + if err != nil { + return nil, err + } + defer did.Release() + cids = append(cids, did.Cid()) + } + + blks, err := g.getBlocks(ctx, cids) + if err != nil { + return nil, fmt.Errorf("getting blocks: %w", err) + } + + nShrs := make([]share.NamespacedRow, len(blks)) + for _, blk := range blks { + data, err := shwap.DataFromBlock(blk) + if err != nil { + return nil, fmt.Errorf("getting row from block: %w", err) + } + + nShrs[int(data.RowIndex)-from] = share.NamespacedRow{ + Shares: data.DataShares, + Proof: &data.DataProof, + } + } + + return nShrs, nil +} + +func (g *Getter) getBlocks(ctx context.Context, cids []cid.Cid) ([]block.Block, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + ses := g.fetch.NewSession(ctx) + // must start getting only after verifiers are registered + blkCh, err := ses.GetBlocks(ctx, cids) + if err != nil { + return nil, fmt.Errorf("fetching blocks: %w", err) + } + // GetBlocks handles ctx and closes blkCh, so we don't have to + blks := make([]block.Block, 0, len(cids)) + for blk := range blkCh { + blks = append(blks, blk) + } + // only persist when all samples received + if len(blks) != len(cids) { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("not all blocks were found") + } + + return blks, nil +} diff --git a/share/shwap/getter/getter_test.go b/share/shwap/getter/getter_test.go new file mode 100644 index 0000000000..fcaf7e9d22 --- /dev/null +++ b/share/shwap/getter/getter_test.go @@ -0,0 +1,225 @@ +package shwap_getter + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/exchange" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + ds_sync "github.com/ipfs/go-datastore/sync" + format "github.com/ipfs/go-ipld-format" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/header/headertest" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/store" + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" +) + +func TestGetter(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + size := 8 + ns := sharetest.RandV0Namespace() + square, root := edstest.RandEDSWithNamespace(t, ns, size*size, size) + hdr := &header.ExtendedHeader{RawHeader: header.RawHeader{Height: 1}, DAH: root} + + store, bstore := edsBlockstore(t) + put(t, store, square, hdr.Height()) + exch := DummySessionExchange{bstore} + get := NewGetter(exch, blockstore.NewBlockstore(datastore.NewMapDatastore())) + + t.Run("GetShares", func(t *testing.T) { + idxs := rand.Perm(int(square.Width() ^ 2))[:10] + shrs, err := get.GetShares(ctx, hdr, idxs...) + assert.NoError(t, err) + + for i, shrs := range shrs { + idx := idxs[i] + x, y := uint(idx)/square.Width(), uint(idx)%square.Width() + cell := square.GetCell(x, y) + ok := bytes.Equal(cell, shrs) + require.True(t, ok) + } + }) + + t.Run("GetShares from empty", func(t *testing.T) { + emptyRoot := da.MinDataAvailabilityHeader() + eh := headertest.RandExtendedHeaderWithRoot(t, &emptyRoot) + + idxs := []int{0, 1, 2, 3} + square := share.EmptyExtendedDataSquare() + shrs, err := get.GetShares(ctx, eh, idxs...) + assert.NoError(t, err) + + for i, shrs := range shrs { + idx := idxs[i] + x, y := uint(idx)/square.Width(), uint(idx)%square.Width() + cell := square.GetCell(x, y) + ok := bytes.Equal(cell, shrs) + require.True(t, ok) + } + }) + + t.Run("GetEDS", func(t *testing.T) { + ctx, cancel := context.WithTimeout(ctx, time.Second) + t.Cleanup(cancel) + + eds, err := get.GetEDS(ctx, hdr) + assert.NoError(t, err) + assert.NotNil(t, eds) + + ok := eds.Equals(square) + assert.True(t, ok) + }) + + t.Run("GetEDS empty", func(t *testing.T) { + ctx, cancel := context.WithTimeout(ctx, time.Second) + t.Cleanup(cancel) + + emptyRoot := da.MinDataAvailabilityHeader() + eh := headertest.RandExtendedHeaderWithRoot(t, &emptyRoot) + + eds, err := get.GetEDS(ctx, eh) + assert.NoError(t, err) + assert.NotNil(t, eds) + + dah, err := share.NewRoot(eds) + require.NoError(t, err) + require.True(t, share.DataHash(dah.Hash()).IsEmptyRoot()) + }) + + t.Run("GetSharesByNamespace", func(t *testing.T) { + nshrs, err := get.GetSharesByNamespace(ctx, hdr, ns) + assert.NoError(t, err) + assert.NoError(t, nshrs.Verify(root, ns)) + assert.NotEmpty(t, nshrs.Flatten()) + + t.Run("NamespaceOutsideOfRoot", func(t *testing.T) { + randNamespace := sharetest.RandV0Namespace() + emptyShares, err := get.GetSharesByNamespace(ctx, hdr, randNamespace) + assert.NoError(t, err) + assert.Empty(t, emptyShares) + assert.NoError(t, emptyShares.Verify(root, randNamespace)) + assert.Empty(t, emptyShares.Flatten()) + }) + + t.Run("NamespaceInsideOfRoot", func(t *testing.T) { + // this test requires a different setup, so we generate a new EDS + square := edstest.RandEDS(t, 8) + root, err := share.NewRoot(square) + require.NoError(t, err) + hdr := &header.ExtendedHeader{RawHeader: header.RawHeader{Height: 3}, DAH: root} + + store, bstore := edsBlockstore(t) + put(t, store, square, hdr.Height()) + exch := &DummySessionExchange{bstore} + get := NewGetter(exch, blockstore.NewBlockstore(datastore.NewMapDatastore())) + + maxNs := nmt.MaxNamespace(root.RowRoots[(len(root.RowRoots))/2-1], share.NamespaceSize) + ns, err := share.Namespace(maxNs).AddInt(-1) + require.NoError(t, err) + require.Len(t, ipld.FilterRootByNamespace(root, ns), 1) + + emptyShares, err := get.GetSharesByNamespace(ctx, hdr, ns) + assert.NoError(t, err) + assert.NotNil(t, emptyShares[0].Proof) + assert.NoError(t, emptyShares.Verify(root, ns)) + assert.Empty(t, emptyShares.Flatten()) + }) + }) +} + +type DummySessionExchange struct { + blockstore.Blockstore +} + +func (e DummySessionExchange) NewSession(context.Context) exchange.Fetcher { + return e +} + +func (e DummySessionExchange) GetBlock(ctx context.Context, k cid.Cid) (blocks.Block, error) { + blk, err := e.Get(ctx, k) + if format.IsNotFound(err) { + return nil, fmt.Errorf("block was not found locally (offline): %w", err) + } + if err != nil { + fmt.Println("ERROR", err) + return nil, err + } + rbcid, err := k.Prefix().Sum(blk.RawData()) + if err != nil { + return nil, err + } + + if !rbcid.Equals(k) { + return nil, blockstore.ErrHashMismatch + } + return blk, err +} + +func (e DummySessionExchange) NotifyNewBlocks(context.Context, ...blocks.Block) error { + return nil +} + +func (e DummySessionExchange) GetBlocks(ctx context.Context, ks []cid.Cid) (<-chan blocks.Block, error) { + out := make(chan blocks.Block) + go func() { + defer close(out) + for _, k := range ks { + hit, err := e.GetBlock(ctx, k) + if err != nil { + select { + case <-ctx.Done(): + return + default: + continue + } + } + select { + case out <- hit: + case <-ctx.Done(): + return + } + } + }() + return out, nil +} + +func (e DummySessionExchange) Close() error { + // NB: exchange doesn't own the blockstore's underlying datastore, so it is + // not responsible for closing it. + return nil +} + +func edsBlockstore(t *testing.T) (*store.Store, blockstore.Blockstore) { + edsStore, err := store.NewStore(store.DefaultParameters(), t.TempDir()) + require.NoError(t, err) + + return edsStore, store.NewBlockstore(edsStore, ds_sync.MutexWrap(datastore.NewMapDatastore())) +} + +func put(t *testing.T, store *store.Store, eds *rsmt2d.ExtendedDataSquare, height uint64) { + dah, err := share.NewRoot(eds) + require.NoError(t, err) + + f, err := store.Put(context.Background(), dah.Hash(), height, eds) + require.NoError(t, err) + f.Close() +} diff --git a/share/shwap/getter/reconstruction.go b/share/shwap/getter/reconstruction.go new file mode 100644 index 0000000000..132b1eeeb0 --- /dev/null +++ b/share/shwap/getter/reconstruction.go @@ -0,0 +1,30 @@ +package shwap_getter + +import ( + "context" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share" +) + +type ReconstructionGetter struct { + retriever *edsRetriver +} + +func NewReconstructionGetter(getter *Getter) *ReconstructionGetter { + return &ReconstructionGetter{retriever: newRetriever(getter)} +} + +func (r ReconstructionGetter) GetShare(ctx context.Context, header *header.ExtendedHeader, row, col int) (share.Share, error) { + return nil, share.ErrOperationNotSupported +} + +func (r ReconstructionGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { + return r.retriever.Retrieve(ctx, header) +} + +func (r ReconstructionGetter) GetSharesByNamespace(ctx context.Context, header *header.ExtendedHeader, namespace share.Namespace) (share.NamespacedShares, error) { + return nil, share.ErrOperationNotSupported +} diff --git a/share/eds/retriever.go b/share/shwap/getter/retriever.go similarity index 50% rename from share/eds/retriever.go rename to share/shwap/getter/retriever.go index c2966c3953..4602ed0820 100644 --- a/share/eds/retriever.go +++ b/share/shwap/getter/retriever.go @@ -1,4 +1,4 @@ -package eds +package shwap_getter import ( "context" @@ -8,29 +8,38 @@ import ( "time" "github.com/ipfs/boxo/blockservice" - "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" - "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/wrapper" - "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/byzantine" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/shwap" ) +// TODO(@walldiss): +// - update comments +// - befp construction should work over share.getter instead of blockservice +// - use single bitswap session for fetching shares +// - don't request repaired shares +// - use enriched logger for session +// - remove per-share tracing +// - remove quadrants struct +// - remove unneeded locks +// - add metrics + var ( log = logging.Logger("share/eds") tracer = otel.Tracer("share/eds") ) -// Retriever retrieves rsmt2d.ExtendedDataSquares from the IPLD network. +// edsRetriver retrieves rsmt2d.ExtendedDataSquares from the IPLD network. // Instead of requesting data 'share by share' it requests data by quadrants // minimizing bandwidth usage in the happy cases. // @@ -40,15 +49,20 @@ var ( // | 2 | 3 | // ---- ---- // -// Retriever randomly picks one of the data square quadrants and tries to request them one by one +// edsRetriver randomly picks one of the data square quadrants and tries to request them one by one // until it is able to reconstruct the whole square. -type Retriever struct { - bServ blockservice.BlockService +type edsRetriver struct { + bServ blockservice.BlockService + getter share.Getter } -// NewRetriever creates a new instance of the Retriever over IPLD BlockService and rmst2d.Codec -func NewRetriever(bServ blockservice.BlockService) *Retriever { - return &Retriever{bServ: bServ} +// newRetriever creates a new instance of the edsRetriver over IPLD BlockService and rmst2d.Codec +func newRetriever(getter *Getter) *edsRetriver { + bServ := blockservice.New(getter.bstore, getter.fetch, blockservice.WithAllowlist(shwap.DefaultAllowlist)) + return &edsRetriver{ + bServ: bServ, + getter: getter, + } } // Retrieve retrieves all the data committed to DataAvailabilityHeader. @@ -57,7 +71,8 @@ func NewRetriever(bServ blockservice.BlockService) *Retriever { // data square and reconstructs the other three quadrants (3/4). If the requested quadrant is not // available within RetrieveQuadrantTimeout, it starts requesting another quadrant until either the // data is reconstructed, context is canceled or ErrByzantine is generated. -func (r *Retriever) Retrieve(ctx context.Context, dah *da.DataAvailabilityHeader) (*rsmt2d.ExtendedDataSquare, error) { +func (r *edsRetriver) Retrieve(ctx context.Context, h *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { + dah := h.DAH ctx, cancel := context.WithCancel(ctx) defer cancel() // cancels all the ongoing requests if reconstruction succeeds early @@ -68,7 +83,7 @@ func (r *Retriever) Retrieve(ctx context.Context, dah *da.DataAvailabilityHeader ) log.Debugw("retrieving data square", "data_hash", dah.String(), "size", len(dah.RowRoots)) - ses, err := r.newSession(ctx, dah) + ses, err := r.newSession(ctx, h) if err != nil { return nil, err } @@ -103,13 +118,9 @@ func (r *Retriever) Retrieve(ctx context.Context, dah *da.DataAvailabilityHeader // quadrant request retries. Also, provides an API // to reconstruct the block once enough shares are fetched. type retrievalSession struct { - dah *da.DataAvailabilityHeader - bget blockservice.BlockGetter + header *header.ExtendedHeader + getter share.Getter - // TODO(@Wondertan): Extract into a separate data structure - // https://github.com/celestiaorg/rsmt2d/issues/135 - squareQuadrants []*quadrant - squareCellsLks [][]sync.Mutex squareCellsCount uint32 squareSig chan struct{} squareDn chan struct{} @@ -120,18 +131,11 @@ type retrievalSession struct { } // newSession creates a new retrieval session and kicks off requesting process. -func (r *Retriever) newSession(ctx context.Context, dah *da.DataAvailabilityHeader) (*retrievalSession, error) { - size := len(dah.RowRoots) +func (r *edsRetriver) newSession(ctx context.Context, h *header.ExtendedHeader) (*retrievalSession, error) { + size := len(h.DAH.RowRoots) treeFn := func(_ rsmt2d.Axis, index uint) rsmt2d.Tree { - // use proofs adder if provided, to cache collected proofs while recomputing the eds - var opts []nmt.Option - visitor := ipld.ProofsAdderFromCtx(ctx).VisitFn() - if visitor != nil { - opts = append(opts, nmt.NodeVisitor(visitor)) - } - - tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(size)/2, index, opts...) + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(size)/2, index) return &tree } @@ -141,17 +145,12 @@ func (r *Retriever) newSession(ctx context.Context, dah *da.DataAvailabilityHead } ses := &retrievalSession{ - dah: dah, - bget: blockservice.NewSession(ctx, r.bServ), - squareQuadrants: newQuadrants(dah), - squareCellsLks: make([][]sync.Mutex, size), - squareSig: make(chan struct{}, 1), - squareDn: make(chan struct{}), - square: square, - span: trace.SpanFromContext(ctx), - } - for i := range ses.squareCellsLks { - ses.squareCellsLks[i] = make([]sync.Mutex, size) + header: h, + getter: r.getter, + squareSig: make(chan struct{}, 1), + squareDn: make(chan struct{}), + square: square, + span: trace.SpanFromContext(ctx), } go ses.request(ctx) @@ -178,12 +177,12 @@ func (rs *retrievalSession) Reconstruct(ctx context.Context) (*rsmt2d.ExtendedDa defer span.End() // and try to repair with what we have - err := rs.square.Repair(rs.dah.RowRoots, rs.dah.ColumnRoots) + err := rs.square.Repair(rs.header.DAH.RowRoots, rs.header.DAH.ColumnRoots) if err != nil { span.RecordError(err) return nil, err } - log.Infow("data square reconstructed", "data_hash", rs.dah.String(), "size", len(rs.dah.RowRoots)) + log.Infow("data square reconstructed", "data_hash", rs.header.DAH.String(), "size", len(rs.header.DAH.RowRoots)) close(rs.squareDn) return rs.square, nil } @@ -211,21 +210,16 @@ func (rs *retrievalSession) Close() error { func (rs *retrievalSession) request(ctx context.Context) { t := time.NewTicker(RetrieveQuadrantTimeout) defer t.Stop() - for retry := 0; retry < len(rs.squareQuadrants); retry++ { - q := rs.squareQuadrants[retry] + for _, q := range newQuadrants() { log.Debugw("requesting quadrant", - "axis", q.source, "x", q.x, "y", q.y, - "size", len(q.roots), ) rs.span.AddEvent("requesting quadrant", trace.WithAttributes( - attribute.Int("axis", int(q.source)), attribute.Int("x", q.x), attribute.Int("y", q.y), - attribute.Int("size", len(q.roots)), )) - rs.doRequest(ctx, q) + rs.requestQuadrant(ctx, q) select { case <-t.C: case <-ctx.Done(): @@ -233,83 +227,72 @@ func (rs *retrievalSession) request(ctx context.Context) { } log.Warnw("quadrant request timeout", "timeout", RetrieveQuadrantTimeout.String(), - "axis", q.source, "x", q.x, "y", q.y, - "size", len(q.roots), ) rs.span.AddEvent("quadrant request timeout", trace.WithAttributes( - attribute.Int("axis", int(q.source)), attribute.Int("x", q.x), attribute.Int("y", q.y), - attribute.Int("size", len(q.roots)), )) } } -// doRequest requests the given quadrant by requesting halves of axis(Row or Col) using GetShares +// requestQuadrant requests the given quadrant by requesting halves of axis(Row or Col) using GetShares // and fills shares into rs.square slice. -func (rs *retrievalSession) doRequest(ctx context.Context, q *quadrant) { - size := len(q.roots) - for i, root := range q.roots { - go func(i int, root cid.Cid) { - // get the root node - nd, err := ipld.GetNode(ctx, rs.bget, root) - if err != nil { - rs.span.RecordError(err, trace.WithAttributes( - attribute.Int("root-index", i), - )) - return - } - // and go get shares of left or the right side of the whole col/row axis - // the left or the right side of the tree represent some portion of the quadrant - // which we put into the rs.square share-by-share by calculating shares' indexes using q.index - ipld.GetShares(ctx, rs.bget, nd.Links()[q.x].Cid, size, func(j int, share share.Share) { - // NOTE: Each share can appear twice here, for a Row and Col, respectively. - // These shares are always equal, and we allow only the first one to be written - // in the square. - // NOTE-2: We may never actually fetch shares from the network *twice*. - // Once a share is downloaded from the network it may be cached on the IPLD(blockservice) level. - // - // calc position of the share - x, y := q.pos(i, j) - // try to lock the share - ok := rs.squareCellsLks[x][y].TryLock() - if !ok { - // if already locked and written - do nothing - return - } - // The R lock here is *not* to protect rs.square from multiple - // concurrent shares writes but to avoid races between share writes and - // repairing attempts. - // Shares are written atomically in their own slice slots and these "writes" do - // not need synchronization! - rs.squareLk.RLock() - defer rs.squareLk.RUnlock() - // the routine could be blocked above for some time during which the square - // might be reconstructed, if so don't write anything and return - if rs.isReconstructed() { - return - } - if err := rs.square.SetCell(uint(x), uint(y), share); err != nil { - // safe to ignore as: - // * share size already verified - // * the same share might come from either Row or Col - return - } - // if we have >= 1/4 of the square we can start trying to Reconstruct - // TODO(@Wondertan): This is not an ideal way to know when to start - // reconstruction and can cause idle reconstruction tries in some cases, - // but it is totally fine for the happy case and for now. - // The earlier we correctly know that we have the full square - the earlier - // we cancel ongoing requests - the less data is being wastedly transferred. - if atomic.AddUint32(&rs.squareCellsCount, 1) >= uint32(size*size) { - select { - case rs.squareSig <- struct{}{}: - default: - } - } - }) - }(i, root) +func (rs *retrievalSession) requestQuadrant(ctx context.Context, q quadrant) { + odsSize := len(rs.header.DAH.RowRoots) / 2 + for x := q.x * odsSize; x < (q.x+1)*odsSize; x++ { + for y := q.y * odsSize; y < (q.y+1)*odsSize; y++ { + go rs.requestCell(ctx, x, y) + } + } +} + +func (rs *retrievalSession) requestCell(ctx context.Context, x, y int) { + share, err := rs.getter.GetShare(ctx, rs.header, x, y) + if err != nil { + log.Debugw("failed to get share", + "height", rs.header.Height, + "x", x, + "y", y, + "err", err, + ) + return + } + + // the routine could be blocked above for some time during which the square + // might be reconstructed, if so don't write anything and return + if rs.isReconstructed() { + return + } + + rs.squareLk.RLock() + defer rs.squareLk.RUnlock() + + if err := rs.square.SetCell(uint(x), uint(y), share); err != nil { + log.Warnw("failed to set cell", + "height", rs.header.Height, + "x", x, + "y", y, + "err", err, + ) + return + } + rs.indicateDone() +} + +func (rs *retrievalSession) indicateDone() { + size := len(rs.header.DAH.RowRoots) / 2 + // if we have >= 1/4 of the square we can start trying to Reconstruct + // TODO(@Wondertan): This is not an ideal way to know when to start + // reconstruction and can cause idle reconstruction tries in some cases, + // but it is totally fine for the happy case and for now. + // The earlier we correctly know that we have the full square - the earlier + // we cancel ongoing requests - the less data is being wastedly transferred. + if atomic.AddUint32(&rs.squareCellsCount, 1) >= uint32(size*size) { + select { + case rs.squareSig <- struct{}{}: + default: + } } } diff --git a/share/shwap/getter/retriever_quadrant.go b/share/shwap/getter/retriever_quadrant.go new file mode 100644 index 0000000000..2fa028b959 --- /dev/null +++ b/share/shwap/getter/retriever_quadrant.go @@ -0,0 +1,44 @@ +package shwap_getter + +import ( + "time" +) + +const ( + // there are always 4 quadrants + numQuadrants = 4 + // blockTime equals to the time with which new blocks are produced in the network. + // TODO(@Wondertan): Here we assume that the block time is a minute, but + // block time is a network wide variable/param that has to be taken from + // a proper place + blockTime = time.Minute +) + +// RetrieveQuadrantTimeout defines how much time edsRetriver waits before +// starting to retrieve another quadrant. +// +// NOTE: +// - The whole data square must be retrieved in less than block time. +// - We have 4 quadrants from two sources(rows, cols) which equals to 8 in total. +var RetrieveQuadrantTimeout = blockTime / numQuadrants * 2 + +type quadrant struct { + // Example coordinates(x;y) of each quadrant + // ------ ------- + // | Q0 | | Q1 | + // |(0;0)| |(1;0)| + // ------ ------- + // | Q2 | | Q3 | + // |(0;1)| |(1;1)| + // ------ ------- + x, y int +} + +// newQuadrants constructs a slice of quadrants. There are always 4 quadrants. +func newQuadrants() []quadrant { + quadrants := make([]quadrant, 0, numQuadrants) + for i := 0; i < numQuadrants; i++ { + quadrants = append(quadrants, quadrant{x: i % 2, y: i / 2}) + } + return quadrants +} diff --git a/share/shwap/getter/retriever_test.go b/share/shwap/getter/retriever_test.go new file mode 100644 index 0000000000..dba857ca8e --- /dev/null +++ b/share/shwap/getter/retriever_test.go @@ -0,0 +1,273 @@ +package shwap_getter + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/byzantine" + "github.com/celestiaorg/celestia-node/share/testing/edstest" +) + +func TestRetriever_Retrieve(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + store, bstore := edsBlockstore(t) + exch := DummySessionExchange{bstore} + getter := NewGetter(exch, bstore) + r := newRetriever(getter) + + height := atomic.NewUint64(1) + type test struct { + name string + squareSize int + } + tests := []test{ + {"1x1(min)", 1}, + {"2x2(med)", 2}, + {"4x4(med)", 4}, + {"8x8(med)", 8}, + {"16x16(med)", 16}, + {"32x32(med)", 32}, + {"64x64(med)", 64}, + {"128x128(max)", share.MaxSquareSize}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + // generate EDS + eds := edstest.RandEDS(t, tc.squareSize) + height := height.Add(1) + put(t, store, eds, height) + + // limit with timeout, specifically retrieval + ctx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + + root, err := share.NewRoot(eds) + require.NoError(t, err) + hdr := &header.ExtendedHeader{RawHeader: header.RawHeader{Height: int64(height)}, DAH: root} + + out, err := r.Retrieve(ctx, hdr) + require.NoError(t, err) + assert.True(t, eds.Equals(out)) + }) + } +} + +func TestRetriever_ByzantineError(t *testing.T) { + const width = 8 + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + store, bstore := edsBlockstore(t) + exch := DummySessionExchange{bstore} + getter := NewGetter(exch, bstore) + r := newRetriever(getter) + + eds := edstest.RandEDS(t, width) + shares := eds.Flattened() + // corrupt shares so that eds erasure coding does not match + copy(shares[14][share.NamespaceSize:], shares[15][share.NamespaceSize:]) + + // store corrupted eds + put(t, store, eds, 1) + + // ensure we rcv an error + root, err := share.NewRoot(eds) + require.NoError(t, err) + hdr := &header.ExtendedHeader{RawHeader: header.RawHeader{Height: 1}, DAH: root} + _, err = r.Retrieve(ctx, hdr) + var errByz *byzantine.ErrByzantine + require.ErrorAs(t, err, &errByz) +} + +// +//func TestRetriever_ByzantineError(t *testing.T) { +// const width = 8 +// ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) +// defer cancel() +// +// bserv := ipld.NewMemBlockservice() +// shares := edstest.RandEDS(t, width).Flattened() +// _, err := ipld.ImportShares(ctx, shares, bserv) +// require.NoError(t, err) +// +// // corrupt shares so that eds erasure coding does not match +// copy(shares[14][share.NamespaceSize:], shares[15][share.NamespaceSize:]) +// +// // import corrupted eds +// batchAdder := ipld.NewNmtNodeAdder(ctx, bserv, ipld.MaxSizeBatchOption(width*2)) +// attackerEDS, err := rsmt2d.ImportExtendedDataSquare( +// shares, +// share.DefaultRSMT2DCodec(), +// wrapper.NewConstructor(uint64(width), +// nmt.NodeVisitor(batchAdder.Visit)), +// ) +// require.NoError(t, err) +// err = batchAdder.Commit() +// require.NoError(t, err) +// +// // ensure we rcv an error +// dah, err := da.NewDataAvailabilityHeader(attackerEDS) +// require.NoError(t, err) +// r := newRetriever(bserv) +// _, err = r.Retrieve(ctx, &dah) +// var errByz *byzantine.ErrByzantine +// require.ErrorAs(t, err, &errByz) +//} +// +//// TestRetriever_MultipleRandQuadrants asserts that reconstruction succeeds +//// when any three random quadrants requested. +//func TestRetriever_MultipleRandQuadrants(t *testing.T) { +// RetrieveQuadrantTimeout = time.Millisecond * 500 +// const squareSize = 32 +// ctx, cancel := context.WithTimeout(context.Background(), time.Minute) +// defer cancel() +// +// bServ := ipld.NewMemBlockservice() +// r := newRetriever(bServ) +// +// // generate EDS +// shares := sharetest.RandShares(t, squareSize*squareSize) +// in, err := ipld.AddShares(ctx, shares, bServ) +// require.NoError(t, err) +// +// dah, err := da.NewDataAvailabilityHeader(in) +// require.NoError(t, err) +// ses, err := r.newSession(ctx, &dah) +// require.NoError(t, err) +// +// // wait until two additional quadrants requested +// // this reliably allows us to reproduce the issue +// time.Sleep(RetrieveQuadrantTimeout * 2) +// // then ensure we have enough shares for reconstruction for slow machines e.g. CI +// <-ses.Done() +// +// _, err = ses.Reconstruct(ctx) +// assert.NoError(t, err) +//} +// +//func TestFraudProofValidation(t *testing.T) { +// ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) +// defer t.Cleanup(cancel) +// bServ := ipld.NewMemBlockservice() +// +// odsSize := []int{2, 4, 16, 32, 64, 128} +// for _, size := range odsSize { +// t.Run(fmt.Sprintf("ods size:%d", size), func(t *testing.T) { +// var errByz *byzantine.ErrByzantine +// faultHeader, err := generateByzantineError(ctx, t, size, bServ) +// require.True(t, errors.As(err, &errByz)) +// +// p := byzantine.CreateBadEncodingProof([]byte("hash"), faultHeader.Height(), errByz) +// err = p.Validate(faultHeader) +// require.NoError(t, err) +// }) +// } +//} +// +//func generateByzantineError( +// ctx context.Context, +// t *testing.T, +// odsSize int, +// bServ blockservice.BlockService, +//) (*header.ExtendedHeader, error) { +// eds := edstest.RandByzantineEDS(t, odsSize) +// err := ipld.ImportEDS(ctx, eds, bServ) +// require.NoError(t, err) +// h := headertest.ExtendedHeaderFromEDS(t, 1, eds) +// _, err = newRetriever(bServ).Retrieve(ctx, h.DAH) +// +// return h, err +//} +// +///* +//BenchmarkBEFPValidation/ods_size:2 31273 38819 ns/op 68052 B/op 366 allocs/op +//BenchmarkBEFPValidation/ods_size:4 14664 80439 ns/op 135892 B/op 894 allocs/op +//BenchmarkBEFPValidation/ods_size:16 2850 386178 ns/op 587890 B/op 4945 allocs/op +//BenchmarkBEFPValidation/ods_size:32 1399 874490 ns/op 1233399 B/op 11284 allocs/op +//BenchmarkBEFPValidation/ods_size:64 619 2047540 ns/op 2578008 B/op 25364 allocs/op +//BenchmarkBEFPValidation/ods_size:128 259 4934375 ns/op 5418406 B/op 56345 allocs/op +//*/ +//func BenchmarkBEFPValidation(b *testing.B) { +// ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) +// defer b.Cleanup(cancel) +// bServ := ipld.NewMemBlockservice() +// r := newRetriever(bServ) +// t := &testing.T{} +// odsSize := []int{2, 4, 16, 32, 64, 128} +// for _, size := range odsSize { +// b.Run(fmt.Sprintf("ods size:%d", size), func(b *testing.B) { +// b.ResetTimer() +// b.StopTimer() +// eds := edstest.RandByzantineEDS(t, size) +// err := ipld.ImportEDS(ctx, eds, bServ) +// require.NoError(t, err) +// h := headertest.ExtendedHeaderFromEDS(t, 1, eds) +// _, err = r.Retrieve(ctx, h.DAH) +// var errByz *byzantine.ErrByzantine +// require.ErrorAs(t, err, &errByz) +// b.StartTimer() +// +// for i := 0; i < b.N; i++ { +// b.ReportAllocs() +// p := byzantine.CreateBadEncodingProof([]byte("hash"), h.Height(), errByz) +// err = p.Validate(h) +// require.NoError(b, err) +// } +// }) +// } +//} +// +///* +//BenchmarkNewErrByzantineData/ods_size:2 29605 38846 ns/op 49518 B/op 579 allocs/op +//BenchmarkNewErrByzantineData/ods_size:4 11380 105302 ns/op 134967 B/op 1571 allocs/op +//BenchmarkNewErrByzantineData/ods_size:16 1902 631086 ns/op 830199 B/op 9601 allocs/op +//BenchmarkNewErrByzantineData/ods_size:32 756 1530985 ns/op 1985272 B/op 22901 allocs/op +//BenchmarkNewErrByzantineData/ods_size:64 340 3445544 ns/op 4767053 B/op 54704 allocs/op +//BenchmarkNewErrByzantineData/ods_size:128 132 8740678 ns/op 11991093 B/op 136584 allocs/op +//*/ +//func BenchmarkNewErrByzantineData(b *testing.B) { +// odsSize := []int{2, 4, 16, 32, 64, 128} +// ctx, cancel := context.WithTimeout(context.Background(), time.Minute) +// defer cancel() +// bServ := ipld.NewMemBlockservice() +// r := newRetriever(bServ) +// t := &testing.T{} +// for _, size := range odsSize { +// b.Run(fmt.Sprintf("ods size:%d", size), func(b *testing.B) { +// b.StopTimer() +// eds := edstest.RandByzantineEDS(t, size) +// err := ipld.ImportEDS(ctx, eds, bServ) +// require.NoError(t, err) +// h := headertest.ExtendedHeaderFromEDS(t, 1, eds) +// ses, err := r.newSession(ctx, h.DAH) +// require.NoError(t, err) +// +// select { +// case <-ctx.Done(): +// b.Fatal(ctx.Err()) +// case <-ses.Done(): +// } +// +// _, err = ses.Reconstruct(ctx) +// assert.NoError(t, err) +// var errByz *rsmt2d.ErrByzantineData +// require.ErrorAs(t, err, &errByz) +// b.StartTimer() +// +// for i := 0; i < b.N; i++ { +// err = byzantine.NewErrByzantine(ctx, bServ, h.DAH, errByz) +// require.NotNil(t, err) +// } +// }) +// } +//} diff --git a/share/shwap/handler.go b/share/shwap/handler.go new file mode 100644 index 0000000000..859c087d23 --- /dev/null +++ b/share/shwap/handler.go @@ -0,0 +1,47 @@ +package shwap + +import ( + "context" + "fmt" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + + "github.com/celestiaorg/celestia-node/share/store/file" +) + +// BlockBuilder is an interface for building response blocks from request and file. +type BlockBuilder interface { + // TODO(@walldiss): don't like this name, but it collides with field name in RowID + GetHeight() uint64 + BlockFromFile(ctx context.Context, file file.EdsFile) (blocks.Block, error) +} + +// BlockBuilderFromCID returns a BlockBuilder from a CID. it acts as multiplexer for +// different block types. +func BlockBuilderFromCID(cid cid.Cid) (BlockBuilder, error) { + switch cid.Type() { + case sampleCodec: + h, err := SampleIDFromCID(cid) + if err != nil { + return nil, fmt.Errorf("while converting CID to SampleID: %w", err) + } + + return h, nil + case rowCodec: + var err error + rid, err := RowIDFromCID(cid) + if err != nil { + return nil, fmt.Errorf("while converting CID to RowID: %w", err) + } + return rid, nil + case dataCodec: + did, err := DataIDFromCID(cid) + if err != nil { + return nil, fmt.Errorf("while converting CID to DataID: %w", err) + } + return did, nil + default: + return nil, fmt.Errorf("unsupported codec") + } +} diff --git a/share/shwap/pb/shwap_pb.pb.go b/share/shwap/pb/shwap_pb.pb.go new file mode 100644 index 0000000000..a0541e8172 --- /dev/null +++ b/share/shwap/pb/shwap_pb.pb.go @@ -0,0 +1,1015 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: share/shwap/pb/shwap_pb.proto + +package shwap_pb + +import ( + fmt "fmt" + pb "github.com/celestiaorg/nmt/pb" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type ProofType int32 + +const ( + ProofType_RowProofType ProofType = 0 + ProofType_ColProofType ProofType = 1 +) + +var ProofType_name = map[int32]string{ + 0: "RowProofType", + 1: "ColProofType", +} + +var ProofType_value = map[string]int32{ + "RowProofType": 0, + "ColProofType": 1, +} + +func (x ProofType) String() string { + return proto.EnumName(ProofType_name, int32(x)) +} + +func (ProofType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_fdfe0676a85dc852, []int{0} +} + +type Row struct { + RowId []byte `protobuf:"bytes,1,opt,name=row_id,json=rowId,proto3" json:"row_id,omitempty"` + RowHalf [][]byte `protobuf:"bytes,2,rep,name=row_half,json=rowHalf,proto3" json:"row_half,omitempty"` +} + +func (m *Row) Reset() { *m = Row{} } +func (m *Row) String() string { return proto.CompactTextString(m) } +func (*Row) ProtoMessage() {} +func (*Row) Descriptor() ([]byte, []int) { + return fileDescriptor_fdfe0676a85dc852, []int{0} +} +func (m *Row) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Row) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Row.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Row) XXX_Merge(src proto.Message) { + xxx_messageInfo_Row.Merge(m, src) +} +func (m *Row) XXX_Size() int { + return m.Size() +} +func (m *Row) XXX_DiscardUnknown() { + xxx_messageInfo_Row.DiscardUnknown(m) +} + +var xxx_messageInfo_Row proto.InternalMessageInfo + +func (m *Row) GetRowId() []byte { + if m != nil { + return m.RowId + } + return nil +} + +func (m *Row) GetRowHalf() [][]byte { + if m != nil { + return m.RowHalf + } + return nil +} + +type Sample struct { + SampleId []byte `protobuf:"bytes,1,opt,name=sample_id,json=sampleId,proto3" json:"sample_id,omitempty"` + SampleShare []byte `protobuf:"bytes,2,opt,name=sample_share,json=sampleShare,proto3" json:"sample_share,omitempty"` + SampleProof *pb.Proof `protobuf:"bytes,3,opt,name=sample_proof,json=sampleProof,proto3" json:"sample_proof,omitempty"` + ProofType ProofType `protobuf:"varint,4,opt,name=proof_type,json=proofType,proto3,enum=ProofType" json:"proof_type,omitempty"` +} + +func (m *Sample) Reset() { *m = Sample{} } +func (m *Sample) String() string { return proto.CompactTextString(m) } +func (*Sample) ProtoMessage() {} +func (*Sample) Descriptor() ([]byte, []int) { + return fileDescriptor_fdfe0676a85dc852, []int{1} +} +func (m *Sample) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Sample) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Sample.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Sample) XXX_Merge(src proto.Message) { + xxx_messageInfo_Sample.Merge(m, src) +} +func (m *Sample) XXX_Size() int { + return m.Size() +} +func (m *Sample) XXX_DiscardUnknown() { + xxx_messageInfo_Sample.DiscardUnknown(m) +} + +var xxx_messageInfo_Sample proto.InternalMessageInfo + +func (m *Sample) GetSampleId() []byte { + if m != nil { + return m.SampleId + } + return nil +} + +func (m *Sample) GetSampleShare() []byte { + if m != nil { + return m.SampleShare + } + return nil +} + +func (m *Sample) GetSampleProof() *pb.Proof { + if m != nil { + return m.SampleProof + } + return nil +} + +func (m *Sample) GetProofType() ProofType { + if m != nil { + return m.ProofType + } + return ProofType_RowProofType +} + +type Data struct { + DataId []byte `protobuf:"bytes,1,opt,name=data_id,json=dataId,proto3" json:"data_id,omitempty"` + DataShares [][]byte `protobuf:"bytes,2,rep,name=data_shares,json=dataShares,proto3" json:"data_shares,omitempty"` + DataProof *pb.Proof `protobuf:"bytes,3,opt,name=data_proof,json=dataProof,proto3" json:"data_proof,omitempty"` +} + +func (m *Data) Reset() { *m = Data{} } +func (m *Data) String() string { return proto.CompactTextString(m) } +func (*Data) ProtoMessage() {} +func (*Data) Descriptor() ([]byte, []int) { + return fileDescriptor_fdfe0676a85dc852, []int{2} +} +func (m *Data) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Data) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Data.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Data) XXX_Merge(src proto.Message) { + xxx_messageInfo_Data.Merge(m, src) +} +func (m *Data) XXX_Size() int { + return m.Size() +} +func (m *Data) XXX_DiscardUnknown() { + xxx_messageInfo_Data.DiscardUnknown(m) +} + +var xxx_messageInfo_Data proto.InternalMessageInfo + +func (m *Data) GetDataId() []byte { + if m != nil { + return m.DataId + } + return nil +} + +func (m *Data) GetDataShares() [][]byte { + if m != nil { + return m.DataShares + } + return nil +} + +func (m *Data) GetDataProof() *pb.Proof { + if m != nil { + return m.DataProof + } + return nil +} + +func init() { + proto.RegisterEnum("ProofType", ProofType_name, ProofType_value) + proto.RegisterType((*Row)(nil), "Row") + proto.RegisterType((*Sample)(nil), "Sample") + proto.RegisterType((*Data)(nil), "Data") +} + +func init() { proto.RegisterFile("share/shwap/pb/shwap_pb.proto", fileDescriptor_fdfe0676a85dc852) } + +var fileDescriptor_fdfe0676a85dc852 = []byte{ + // 314 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0xc1, 0x4a, 0xf3, 0x40, + 0x14, 0x85, 0x33, 0x6d, 0xff, 0xb4, 0xb9, 0x0d, 0xfd, 0xcb, 0x80, 0x38, 0x2a, 0x8e, 0xb1, 0xab, + 0xe8, 0x22, 0x81, 0xba, 0x70, 0xaf, 0x2e, 0xec, 0x4e, 0x52, 0xf7, 0x65, 0x42, 0x52, 0x2a, 0x44, + 0x66, 0x98, 0x04, 0x86, 0xbe, 0x85, 0x6f, 0xe1, 0xab, 0xb8, 0xec, 0xd2, 0xa5, 0xb4, 0x2f, 0x22, + 0x73, 0x93, 0x26, 0x3b, 0x77, 0xe7, 0x7c, 0xe7, 0x0e, 0x7c, 0x21, 0x70, 0x59, 0x6e, 0x84, 0xce, + 0xe3, 0x72, 0x63, 0x84, 0x8a, 0x55, 0x5a, 0x87, 0x95, 0x4a, 0x23, 0xa5, 0x65, 0x25, 0xcf, 0x27, + 0x2a, 0x8d, 0x95, 0x96, 0x72, 0x5d, 0xf7, 0xd9, 0x3d, 0xf4, 0x13, 0x69, 0xe8, 0x09, 0xb8, 0x5a, + 0x9a, 0xd5, 0x5b, 0xc6, 0x48, 0x40, 0x42, 0x3f, 0xf9, 0xa7, 0xa5, 0x59, 0x64, 0xf4, 0x0c, 0x46, + 0x16, 0x6f, 0x44, 0xb1, 0x66, 0xbd, 0xa0, 0x1f, 0xfa, 0xc9, 0x50, 0x4b, 0xf3, 0x2c, 0x8a, 0xf5, + 0xec, 0x93, 0x80, 0xbb, 0x14, 0xef, 0xaa, 0xc8, 0xe9, 0x05, 0x78, 0x25, 0xa6, 0xee, 0xfd, 0xa8, + 0x06, 0x8b, 0x8c, 0x5e, 0x83, 0xdf, 0x8c, 0x28, 0xc6, 0x7a, 0xb8, 0x8f, 0x6b, 0xb6, 0xb4, 0x88, + 0xce, 0xdb, 0x13, 0x34, 0x63, 0xfd, 0x80, 0x84, 0xe3, 0xf9, 0xff, 0xa8, 0xf1, 0x4c, 0xa3, 0x17, + 0x1b, 0x8e, 0x6f, 0xb0, 0xd0, 0x1b, 0x00, 0x9c, 0x57, 0xd5, 0x56, 0xe5, 0x6c, 0x10, 0x90, 0x70, + 0x32, 0x87, 0xfa, 0xf0, 0x75, 0xab, 0xf2, 0xc4, 0x53, 0xc7, 0x38, 0x53, 0x30, 0x78, 0x12, 0x95, + 0xa0, 0xa7, 0x30, 0xcc, 0x44, 0x25, 0x3a, 0x49, 0xd7, 0xd6, 0x45, 0x46, 0xaf, 0x60, 0x8c, 0x03, + 0x0a, 0x96, 0xcd, 0x87, 0x82, 0x45, 0xe8, 0x57, 0xd2, 0x08, 0xb0, 0xfd, 0xad, 0xe7, 0xd9, 0x13, + 0x8c, 0xb7, 0x31, 0x78, 0xad, 0x09, 0x9d, 0x82, 0x9f, 0x48, 0xd3, 0xf6, 0xa9, 0x63, 0xc9, 0xa3, + 0x2c, 0x3a, 0x42, 0x1e, 0xd8, 0xd7, 0x9e, 0x93, 0xdd, 0x9e, 0x93, 0x9f, 0x3d, 0x27, 0x1f, 0x07, + 0xee, 0xec, 0x0e, 0xdc, 0xf9, 0x3e, 0x70, 0x27, 0x75, 0xf1, 0x37, 0xdd, 0xfd, 0x06, 0x00, 0x00, + 0xff, 0xff, 0x3b, 0x95, 0x2f, 0xb8, 0xd7, 0x01, 0x00, 0x00, +} + +func (m *Row) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Row) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Row) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.RowHalf) > 0 { + for iNdEx := len(m.RowHalf) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.RowHalf[iNdEx]) + copy(dAtA[i:], m.RowHalf[iNdEx]) + i = encodeVarintShwapPb(dAtA, i, uint64(len(m.RowHalf[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.RowId) > 0 { + i -= len(m.RowId) + copy(dAtA[i:], m.RowId) + i = encodeVarintShwapPb(dAtA, i, uint64(len(m.RowId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Sample) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Sample) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Sample) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.ProofType != 0 { + i = encodeVarintShwapPb(dAtA, i, uint64(m.ProofType)) + i-- + dAtA[i] = 0x20 + } + if m.SampleProof != nil { + { + size, err := m.SampleProof.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShwapPb(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + if len(m.SampleShare) > 0 { + i -= len(m.SampleShare) + copy(dAtA[i:], m.SampleShare) + i = encodeVarintShwapPb(dAtA, i, uint64(len(m.SampleShare))) + i-- + dAtA[i] = 0x12 + } + if len(m.SampleId) > 0 { + i -= len(m.SampleId) + copy(dAtA[i:], m.SampleId) + i = encodeVarintShwapPb(dAtA, i, uint64(len(m.SampleId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Data) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Data) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Data) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.DataProof != nil { + { + size, err := m.DataProof.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShwapPb(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + if len(m.DataShares) > 0 { + for iNdEx := len(m.DataShares) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.DataShares[iNdEx]) + copy(dAtA[i:], m.DataShares[iNdEx]) + i = encodeVarintShwapPb(dAtA, i, uint64(len(m.DataShares[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.DataId) > 0 { + i -= len(m.DataId) + copy(dAtA[i:], m.DataId) + i = encodeVarintShwapPb(dAtA, i, uint64(len(m.DataId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintShwapPb(dAtA []byte, offset int, v uint64) int { + offset -= sovShwapPb(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *Row) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.RowId) + if l > 0 { + n += 1 + l + sovShwapPb(uint64(l)) + } + if len(m.RowHalf) > 0 { + for _, b := range m.RowHalf { + l = len(b) + n += 1 + l + sovShwapPb(uint64(l)) + } + } + return n +} + +func (m *Sample) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SampleId) + if l > 0 { + n += 1 + l + sovShwapPb(uint64(l)) + } + l = len(m.SampleShare) + if l > 0 { + n += 1 + l + sovShwapPb(uint64(l)) + } + if m.SampleProof != nil { + l = m.SampleProof.Size() + n += 1 + l + sovShwapPb(uint64(l)) + } + if m.ProofType != 0 { + n += 1 + sovShwapPb(uint64(m.ProofType)) + } + return n +} + +func (m *Data) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.DataId) + if l > 0 { + n += 1 + l + sovShwapPb(uint64(l)) + } + if len(m.DataShares) > 0 { + for _, b := range m.DataShares { + l = len(b) + n += 1 + l + sovShwapPb(uint64(l)) + } + } + if m.DataProof != nil { + l = m.DataProof.Size() + n += 1 + l + sovShwapPb(uint64(l)) + } + return n +} + +func sovShwapPb(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozShwapPb(x uint64) (n int) { + return sovShwapPb(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *Row) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Row: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Row: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RowId", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShwapPb + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShwapPb + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RowId = append(m.RowId[:0], dAtA[iNdEx:postIndex]...) + if m.RowId == nil { + m.RowId = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RowHalf", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShwapPb + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShwapPb + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RowHalf = append(m.RowHalf, make([]byte, postIndex-iNdEx)) + copy(m.RowHalf[len(m.RowHalf)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShwapPb(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShwapPb + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Sample) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Sample: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Sample: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SampleId", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShwapPb + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShwapPb + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SampleId = append(m.SampleId[:0], dAtA[iNdEx:postIndex]...) + if m.SampleId == nil { + m.SampleId = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SampleShare", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShwapPb + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShwapPb + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SampleShare = append(m.SampleShare[:0], dAtA[iNdEx:postIndex]...) + if m.SampleShare == nil { + m.SampleShare = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SampleProof", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShwapPb + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShwapPb + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SampleProof == nil { + m.SampleProof = &pb.Proof{} + } + if err := m.SampleProof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProofType", wireType) + } + m.ProofType = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProofType |= ProofType(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipShwapPb(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShwapPb + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Data) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Data: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Data: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DataId", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShwapPb + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShwapPb + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DataId = append(m.DataId[:0], dAtA[iNdEx:postIndex]...) + if m.DataId == nil { + m.DataId = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DataShares", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShwapPb + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShwapPb + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DataShares = append(m.DataShares, make([]byte, postIndex-iNdEx)) + copy(m.DataShares[len(m.DataShares)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DataProof", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwapPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShwapPb + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShwapPb + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.DataProof == nil { + m.DataProof = &pb.Proof{} + } + if err := m.DataProof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShwapPb(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShwapPb + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipShwapPb(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowShwapPb + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowShwapPb + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowShwapPb + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthShwapPb + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupShwapPb + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthShwapPb + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthShwapPb = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowShwapPb = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupShwapPb = fmt.Errorf("proto: unexpected end of group") +) diff --git a/share/shwap/pb/shwap_pb.proto b/share/shwap/pb/shwap_pb.proto new file mode 100644 index 0000000000..ab5b161ca5 --- /dev/null +++ b/share/shwap/pb/shwap_pb.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +import "pb/proof.proto"; // celestiaorg/nmt/pb/proof.proto + +message Row { + bytes row_id = 1; + repeated bytes row_half = 2; +} + +enum ProofType { + RowProofType = 0; + ColProofType = 1; +} + +message Sample { + bytes sample_id = 1; + bytes sample_share = 2; + proof.pb.Proof sample_proof = 3; + ProofType proof_type = 4; +} + +message Data { + bytes data_id = 1; + repeated bytes data_shares = 2; + proof.pb.Proof data_proof = 3; +} diff --git a/share/shwap/row.go b/share/shwap/row.go new file mode 100644 index 0000000000..266fef4d82 --- /dev/null +++ b/share/shwap/row.go @@ -0,0 +1,126 @@ +package shwap + +import ( + "bytes" + "fmt" + + blocks "github.com/ipfs/go-block-format" + + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + shwappb "github.com/celestiaorg/celestia-node/share/shwap/pb" +) + +// Row represents a Row of an EDS. +type Row struct { + RowID + + // RowShares is the original non erasure-coded half of the Row. + RowShares []share.Share +} + +// NewRow constructs a new Row. +func NewRow(id RowID, axisHalf []share.Share) *Row { + return &Row{ + RowID: id, + RowShares: axisHalf, + } +} + +// NewRowFromEDS constructs a new Row from the given EDS. +func NewRowFromEDS( + height uint64, + rowIdx int, + square *rsmt2d.ExtendedDataSquare, +) (*Row, error) { + sqrLn := int(square.Width()) + axisHalf := square.Row(uint(rowIdx))[:sqrLn/2] + + root, err := share.NewRoot(square) + if err != nil { + return nil, err + } + + id, err := NewRowID(height, uint16(rowIdx), root) + if err != nil { + return nil, err + } + + return NewRow(id, axisHalf), nil +} + +// RowFromBlock converts blocks.Block into Row. +func RowFromBlock(blk blocks.Block) (*Row, error) { + if err := validateCID(blk.Cid()); err != nil { + return nil, err + } + return RowFromBinary(blk.RawData()) +} + +// IPLDBlock converts Row to an IPLD block for Bitswap compatibility. +func (r *Row) IPLDBlock() (blocks.Block, error) { + data, err := r.MarshalBinary() + if err != nil { + return nil, err + } + + return blocks.NewBlockWithCid(data, r.Cid()) +} + +// MarshalBinary marshals Row to binary. +func (r *Row) MarshalBinary() ([]byte, error) { + return (&shwappb.Row{ + RowId: r.RowID.MarshalBinary(), + RowHalf: r.RowShares, + }).Marshal() +} + +// RowFromBinary unmarshal Row from binary. +func RowFromBinary(data []byte) (*Row, error) { + proto := &shwappb.Row{} + if err := proto.Unmarshal(data); err != nil { + return nil, err + } + + rid, err := RowIDFromBinary(proto.RowId) + if err != nil { + return nil, err + } + return NewRow(rid, proto.RowHalf), nil +} + +// Verify validates Row's fields and verifies Row inclusion. +func (r *Row) Verify(root *share.Root) error { + if err := r.RowID.Verify(root); err != nil { + return err + } + + encoded, err := share.DefaultRSMT2DCodec().Encode(r.RowShares) + if err != nil { + return fmt.Errorf("while decoding erasure coded half: %w", err) + } + // TODO: encoded already contains all the shares initially [-len(RowShares):] + r.RowShares = append(r.RowShares, encoded...) + + sqrLn := uint64(len(r.RowShares) / 2) + tree := wrapper.NewErasuredNamespacedMerkleTree(sqrLn, uint(r.RowID.RowIndex)) + for _, shr := range r.RowShares { + err := tree.Push(shr) + if err != nil { + return fmt.Errorf("while pushing shares to NMT: %w", err) + } + } + + rowRoot, err := tree.Root() + if err != nil { + return fmt.Errorf("while computing NMT root: %w", err) + } + + if !bytes.Equal(root.RowRoots[r.RowIndex], rowRoot) { + return fmt.Errorf("invalid RowHash: %X != %X", root, root.RowRoots[r.RowIndex]) + } + + return nil +} diff --git a/share/shwap/row_hasher.go b/share/shwap/row_hasher.go new file mode 100644 index 0000000000..07ecdcc1d4 --- /dev/null +++ b/share/shwap/row_hasher.go @@ -0,0 +1,60 @@ +package shwap + +import ( + "crypto/sha256" + "fmt" +) + +// RowHasher implements hash.Hash interface for Row. +type RowHasher struct { + data []byte +} + +// Write expects a marshaled Row to validate. +func (h *RowHasher) Write(data []byte) (int, error) { + row, err := RowFromBinary(data) + if err != nil { + err = fmt.Errorf("unmarshaling Row: %w", err) + log.Error(err) + return 0, err + } + + root, err := getRoot(row.RowID) + if err != nil { + err = fmt.Errorf("getting root: %w", err) + return 0, err + } + + if err := row.Verify(root); err != nil { + err = fmt.Errorf("verifying Data: %w", err) + log.Error(err) + return 0, err + } + + h.data = data + return len(data), nil +} + +// Sum returns the "multihash" of the RowID. +func (h *RowHasher) Sum([]byte) []byte { + if h.data == nil { + return nil + } + const pbOffset = 2 + return h.data[pbOffset : RowIDSize+pbOffset] +} + +// Reset resets the Hash to its initial state. +func (h *RowHasher) Reset() { + h.data = nil +} + +// Size returns the number of bytes Sum will return. +func (h *RowHasher) Size() int { + return RowIDSize +} + +// BlockSize returns the hash's underlying block size. +func (h *RowHasher) BlockSize() int { + return sha256.BlockSize +} diff --git a/share/shwap/row_hasher_test.go b/share/shwap/row_hasher_test.go new file mode 100644 index 0000000000..cf0f109d10 --- /dev/null +++ b/share/shwap/row_hasher_test.go @@ -0,0 +1,42 @@ +package shwap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/testing/edstest" +) + +func TestRowHasher(t *testing.T) { + hasher := &RowHasher{} + + _, err := hasher.Write([]byte("hello")) + assert.Error(t, err) + + square := edstest.RandEDS(t, 2) + root, err := share.NewRoot(square) + require.NoError(t, err) + + row, err := NewRowFromEDS(2, 1, square) + require.NoError(t, err) + + globalRootsCache.Store(row.RowID, root) + + data, err := row.MarshalBinary() + require.NoError(t, err) + + n, err := hasher.Write(data) + require.NoError(t, err) + assert.EqualValues(t, len(data), n) + + digest := hasher.Sum(nil) + id := row.RowID.MarshalBinary() + assert.EqualValues(t, id, digest) + + hasher.Reset() + digest = hasher.Sum(nil) + assert.NotEqualValues(t, digest, id) +} diff --git a/share/shwap/row_id.go b/share/shwap/row_id.go new file mode 100644 index 0000000000..79e61b38f7 --- /dev/null +++ b/share/shwap/row_id.go @@ -0,0 +1,136 @@ +package shwap + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/store/file" +) + +// RowIDSize is the size of the RowID in bytes +const RowIDSize = EdsIDSize + 2 + +// RowID is an unique identifier of a Row. +type RowID struct { + EdsID + + // RowIndex is the index of the axis(row, col) in the data square + RowIndex uint16 +} + +// NewRowID constructs a new RowID. +func NewRowID(height uint64, rowIdx uint16, root *share.Root) (RowID, error) { + rid := RowID{ + EdsID: EdsID{ + Height: height, + }, + RowIndex: rowIdx, + } + + // Store the root in the cache for verification later + globalRootsCache.Store(rid, root) + return rid, rid.Verify(root) +} + +// RowIDFromCID coverts CID to RowID. +func RowIDFromCID(cid cid.Cid) (id RowID, err error) { + if err = validateCID(cid); err != nil { + return id, err + } + + rid, err := RowIDFromBinary(cid.Hash()[mhPrefixSize:]) + if err != nil { + return id, fmt.Errorf("while unmarhaling RowID: %w", err) + } + return rid, nil +} + +// Cid returns RowID encoded as CID. +func (rid RowID) Cid() cid.Cid { + data := rid.MarshalBinary() + + buf, err := mh.Encode(data, rowMultihashCode) + if err != nil { + panic(fmt.Errorf("encoding RowID as CID: %w", err)) + } + + return cid.NewCidV1(rowCodec, buf) +} + +// MarshalBinary encodes RowID into binary form. +func (rid RowID) MarshalBinary() []byte { + data := make([]byte, 0, RowIDSize) + return rid.appendTo(data) +} + +// RowIDFromBinary decodes RowID from binary form. +func RowIDFromBinary(data []byte) (RowID, error) { + var rid RowID + if len(data) != RowIDSize { + return rid, fmt.Errorf("invalid RowID data length: %d != %d", len(data), RowIDSize) + } + eid, err := EdsIDFromBinary(data[:EdsIDSize]) + if err != nil { + return rid, fmt.Errorf("while decoding EdsID: %w", err) + } + rid.EdsID = eid + return rid, binary.Read(bytes.NewReader(data[EdsIDSize:]), binary.BigEndian, &rid.RowIndex) +} + +// Verify verifies RowID fields. +func (rid RowID) Verify(root *share.Root) error { + if err := rid.EdsID.Verify(root); err != nil { + return err + } + + sqrLn := len(root.RowRoots) + if int(rid.RowIndex) >= sqrLn { + return fmt.Errorf("RowIndex exceeds square size: %d >= %d", rid.RowIndex, sqrLn) + } + + return nil +} + +// BlockFromFile returns the IPLD block of the RowID from the given file. +func (rid RowID) BlockFromFile(ctx context.Context, f file.EdsFile) (blocks.Block, error) { + axisHalf, err := f.AxisHalf(ctx, rsmt2d.Row, int(rid.RowIndex)) + if err != nil { + return nil, fmt.Errorf("while getting AxisHalf: %w", err) + } + + shares := axisHalf.Shares + // If it's a parity axis, we need to get the left half of the shares + if axisHalf.IsParity { + axis, err := axisHalf.Extended() + if err != nil { + return nil, fmt.Errorf("while getting extended shares: %w", err) + } + shares = axis[:len(axis)/2] + } + + s := NewRow(rid, shares) + blk, err := s.IPLDBlock() + if err != nil { + return nil, fmt.Errorf("while coverting to IPLD block: %w", err) + } + return blk, nil +} + +// Release releases the verifier of the RowID. +func (rid RowID) Release() { + globalRootsCache.Delete(rid) +} + +func (rid RowID) appendTo(data []byte) []byte { + data = rid.EdsID.appendTo(data) + return binary.BigEndian.AppendUint16(data, rid.RowIndex) +} diff --git a/share/shwap/row_id_test.go b/share/shwap/row_id_test.go new file mode 100644 index 0000000000..36f399cf7b --- /dev/null +++ b/share/shwap/row_id_test.go @@ -0,0 +1,33 @@ +package shwap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/testing/edstest" +) + +func TestRowID(t *testing.T) { + square := edstest.RandEDS(t, 2) + root, err := share.NewRoot(square) + require.NoError(t, err) + + id, err := NewRowID(2, 1, root) + require.NoError(t, err) + + cid := id.Cid() + assert.EqualValues(t, rowCodec, cid.Prefix().Codec) + assert.EqualValues(t, rowMultihashCode, cid.Prefix().MhType) + assert.EqualValues(t, RowIDSize, cid.Prefix().MhLength) + + data := id.MarshalBinary() + idOut, err := RowIDFromBinary(data) + require.NoError(t, err) + assert.EqualValues(t, id, idOut) + + err = idOut.Verify(root) + require.NoError(t, err) +} diff --git a/share/shwap/row_test.go b/share/shwap/row_test.go new file mode 100644 index 0000000000..0f5103b2d9 --- /dev/null +++ b/share/shwap/row_test.go @@ -0,0 +1,34 @@ +package shwap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/testing/edstest" +) + +func TestRow(t *testing.T) { + square := edstest.RandEDS(t, 8) + root, err := share.NewRoot(square) + require.NoError(t, err) + + row, err := NewRowFromEDS(1, 2, square) + require.NoError(t, err) + + data, err := row.MarshalBinary() + require.NoError(t, err) + + blk, err := row.IPLDBlock() + require.NoError(t, err) + assert.EqualValues(t, blk.Cid(), row.Cid()) + + rowOut, err := RowFromBinary(data) + require.NoError(t, err) + assert.EqualValues(t, row, rowOut) + + err = rowOut.Verify(root) + require.NoError(t, err) +} diff --git a/share/shwap/sample.go b/share/shwap/sample.go new file mode 100644 index 0000000000..6d14680883 --- /dev/null +++ b/share/shwap/sample.go @@ -0,0 +1,180 @@ +package shwap + +import ( + "errors" + "fmt" + + blocks "github.com/ipfs/go-block-format" + + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" + nmtpb "github.com/celestiaorg/nmt/pb" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + shwappb "github.com/celestiaorg/celestia-node/share/shwap/pb" +) + +// SampleProofType is either row or column proven Sample. +type SampleProofType = rsmt2d.Axis + +const ( + // RowProofType is a sample proven via row root of the square. + RowProofType = rsmt2d.Row + // ColProofType is a sample proven via column root of the square. + ColProofType = rsmt2d.Col +) + +// Sample represents a sample of an NMT in EDS. +type Sample struct { + SampleID + + // SampleProofType of the Sample + SampleProofType SampleProofType + // SampleProof of SampleShare inclusion in the NMT + SampleProof nmt.Proof + // SampleShare is a share being sampled + SampleShare share.Share +} + +// NewSample constructs a new Sample. +func NewSample(id SampleID, shr share.Share, proof nmt.Proof, proofTp SampleProofType) *Sample { + return &Sample{ + SampleID: id, + SampleProofType: proofTp, + SampleProof: proof, + SampleShare: shr, + } +} + +// NewSampleFromEDS samples the EDS and constructs a new row-proven Sample. +func NewSampleFromEDS( + proofType SampleProofType, + smplIdx int, + square *rsmt2d.ExtendedDataSquare, + height uint64, +) (*Sample, error) { + root, err := share.NewRoot(square) + if err != nil { + return nil, err + } + + id, err := NewSampleID(height, smplIdx, root) + if err != nil { + return nil, err + } + + sqrLn := int(square.Width()) + rowIdx, shrIdx := uint16(smplIdx/sqrLn), uint16(smplIdx%sqrLn) + + // TODO(@Wondertan): Should be an rsmt2d method + var shrs [][]byte + switch proofType { + case rsmt2d.Row: + shrs = square.Row(uint(rowIdx)) + case rsmt2d.Col: + rowIdx, shrIdx = shrIdx, rowIdx + shrs = square.Col(uint(rowIdx)) + default: + panic("invalid axis") + } + + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(sqrLn/2), uint(rowIdx)) + for _, shr := range shrs { + err := tree.Push(shr) + if err != nil { + return nil, fmt.Errorf("while pushing shares to NMT: %w", err) + } + } + + prf, err := tree.ProveRange(int(shrIdx), int(shrIdx+1)) + if err != nil { + return nil, fmt.Errorf("while proving range share over NMT: %w", err) + } + + return NewSample(id, shrs[shrIdx], prf, proofType), nil +} + +// SampleFromBlock converts blocks.Block into Sample. +func SampleFromBlock(blk blocks.Block) (*Sample, error) { + if err := validateCID(blk.Cid()); err != nil { + return nil, err + } + return SampleFromBinary(blk.RawData()) +} + +// IPLDBlock converts Sample to an IPLD block for Bitswap compatibility. +func (s *Sample) IPLDBlock() (blocks.Block, error) { + data, err := s.MarshalBinary() + if err != nil { + return nil, err + } + + return blocks.NewBlockWithCid(data, s.Cid()) +} + +// MarshalBinary marshals Sample to binary. +func (s *Sample) MarshalBinary() ([]byte, error) { + id := s.SampleID.MarshalBinary() + proof := &nmtpb.Proof{} + proof.Nodes = s.SampleProof.Nodes() + proof.End = int64(s.SampleProof.End()) + proof.Start = int64(s.SampleProof.Start()) + proof.IsMaxNamespaceIgnored = s.SampleProof.IsMaxNamespaceIDIgnored() + proof.LeafHash = s.SampleProof.LeafHash() + + return (&shwappb.Sample{ + SampleId: id, + ProofType: shwappb.ProofType(s.SampleProofType), + SampleProof: proof, + SampleShare: s.SampleShare, + }).Marshal() +} + +// SampleFromBinary unmarshal Sample from binary. +func SampleFromBinary(data []byte) (*Sample, error) { + proto := &shwappb.Sample{} + if err := proto.Unmarshal(data); err != nil { + return nil, err + } + + sid, err := SampleIdFromBinary(proto.SampleId) + if err != nil { + return nil, err + } + + return &Sample{ + SampleID: sid, + SampleProofType: SampleProofType(proto.ProofType), + SampleProof: nmt.ProtoToProof(*proto.SampleProof), + SampleShare: proto.SampleShare, + }, nil +} + +// Verify validates Sample's fields and verifies SampleShare inclusion. +func (s *Sample) Verify(root *share.Root) error { + if err := s.SampleID.Verify(root); err != nil { + return err + } + + if s.SampleProofType != RowProofType && s.SampleProofType != ColProofType { + return fmt.Errorf("invalid SampleProofType: %d", s.SampleProofType) + } + + sqrLn := len(root.RowRoots) + namespace := share.ParitySharesNamespace + if int(s.RowIndex) < sqrLn/2 && int(s.ShareIndex) < sqrLn/2 { + namespace = share.GetNamespace(s.SampleShare) + } + + rootHash := root.RowRoots[s.RowIndex] + if s.SampleProofType == ColProofType { + rootHash = root.ColumnRoots[s.ShareIndex] + } + + if !s.SampleProof.VerifyInclusion(hashFn(), namespace.ToNMT(), [][]byte{s.SampleShare}, rootHash) { + return errors.New("invalid Sample") + } + + return nil +} diff --git a/share/shwap/sample_hasher.go b/share/shwap/sample_hasher.go new file mode 100644 index 0000000000..7d5cdb7f30 --- /dev/null +++ b/share/shwap/sample_hasher.go @@ -0,0 +1,60 @@ +package shwap + +import ( + "crypto/sha256" + "fmt" +) + +// SampleHasher implements hash.Hash interface for Sample. +type SampleHasher struct { + data []byte +} + +// Write expects a marshaled Sample to validate. +func (h *SampleHasher) Write(data []byte) (int, error) { + s, err := SampleFromBinary(data) + if err != nil { + err = fmt.Errorf("unmarshaling Sample: %w", err) + log.Error(err) + return 0, err + } + + root, err := getRoot(s.SampleID) + if err != nil { + err = fmt.Errorf("getting root: %w", err) + return 0, err + } + + if err := s.Verify(root); err != nil { + err = fmt.Errorf("verifying Data: %w", err) + log.Error(err) + return 0, err + } + + h.data = data + return len(data), nil +} + +// Sum returns the "multihash" of the SampleID. +func (h *SampleHasher) Sum([]byte) []byte { + if h.data == nil { + return nil + } + const pbOffset = 2 + return h.data[pbOffset : SampleIDSize+pbOffset] +} + +// Reset resets the Hash to its initial state. +func (h *SampleHasher) Reset() { + h.data = nil +} + +// Size returns the number of bytes Sum will return. +func (h *SampleHasher) Size() int { + return SampleIDSize +} + +// BlockSize returns the hash's underlying block size. +func (h *SampleHasher) BlockSize() int { + return sha256.BlockSize +} diff --git a/share/shwap/sample_hasher_test.go b/share/shwap/sample_hasher_test.go new file mode 100644 index 0000000000..f2448507a4 --- /dev/null +++ b/share/shwap/sample_hasher_test.go @@ -0,0 +1,42 @@ +package shwap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/testing/edstest" +) + +func TestSampleHasher(t *testing.T) { + hasher := &SampleHasher{} + + _, err := hasher.Write([]byte("hello")) + assert.Error(t, err) + + square := edstest.RandEDS(t, 2) + root, err := share.NewRoot(square) + require.NoError(t, err) + + sample, err := NewSampleFromEDS(RowProofType, 10, square, 1) + require.NoError(t, err) + + globalRootsCache.Store(sample.SampleID, root) + + data, err := sample.MarshalBinary() + require.NoError(t, err) + + n, err := hasher.Write(data) + require.NoError(t, err) + assert.EqualValues(t, len(data), n) + + digest := hasher.Sum(nil) + id := sample.SampleID.MarshalBinary() + assert.EqualValues(t, id, digest) + + hasher.Reset() + digest = hasher.Sum(nil) + assert.NotEqualValues(t, digest, id) +} diff --git a/share/shwap/sample_id.go b/share/shwap/sample_id.go new file mode 100644 index 0000000000..96a6e9ace2 --- /dev/null +++ b/share/shwap/sample_id.go @@ -0,0 +1,131 @@ +package shwap + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/store/file" +) + +// SampleIDSize is the size of the SampleID in bytes +const SampleIDSize = RowIDSize + 2 + +// SampleID is an unique identifier of a Sample. +type SampleID struct { + RowID + + // ShareIndex is the index of the sampled share in the Row + ShareIndex uint16 +} + +// NewSampleID constructs a new SampleID. +func NewSampleID(height uint64, smplIdx int, root *share.Root) (SampleID, error) { + sqrLn := len(root.RowRoots) + rowIdx, shrIdx := uint16(smplIdx/sqrLn), uint16(smplIdx%sqrLn) + sid := SampleID{ + RowID: RowID{ + EdsID: EdsID{ + Height: height, + }, + RowIndex: rowIdx, + }, + ShareIndex: shrIdx, + } + + // Store the root in the cache for verification later + globalRootsCache.Store(sid, root) + return sid, sid.Verify(root) +} + +// SampleIDFromCID coverts CID to SampleID. +func SampleIDFromCID(cid cid.Cid) (id SampleID, err error) { + if err = validateCID(cid); err != nil { + return id, err + } + + id, err = SampleIdFromBinary(cid.Hash()[mhPrefixSize:]) + if err != nil { + return id, fmt.Errorf("while unmarhaling SampleID: %w", err) + } + + return id, nil +} + +// Cid returns SampleID encoded as CID. +func (sid SampleID) Cid() cid.Cid { + // avoid using proto serialization for CID as it's not deterministic + data := sid.MarshalBinary() + + buf, err := mh.Encode(data, sampleMultihashCode) + if err != nil { + panic(fmt.Errorf("encoding SampleID as CID: %w", err)) + } + + return cid.NewCidV1(sampleCodec, buf) +} + +// MarshalBinary encodes SampleID into binary form. +// NOTE: Proto is avoided because +// * Its size is not deterministic which is required for IPLD. +// * No support for uint16 +func (sid SampleID) MarshalBinary() []byte { + data := make([]byte, 0, SampleIDSize) + return sid.appendTo(data) +} + +// SampleIdFromBinary decodes SampleID from binary form. +func SampleIdFromBinary(data []byte) (SampleID, error) { + var sid SampleID + if len(data) != SampleIDSize { + return sid, fmt.Errorf("invalid SampleID data length: %d != %d", len(data), SampleIDSize) + } + + rid, err := RowIDFromBinary(data[:RowIDSize]) + if err != nil { + return sid, fmt.Errorf("while decoding RowID: %w", err) + } + sid.RowID = rid + return sid, binary.Read(bytes.NewReader(data[RowIDSize:]), binary.BigEndian, &sid.ShareIndex) +} + +// Verify verifies SampleID fields. +func (sid SampleID) Verify(root *share.Root) error { + sqrLn := len(root.ColumnRoots) + if int(sid.ShareIndex) >= sqrLn { + return fmt.Errorf("ShareIndex exceeds square size: %d >= %d", sid.ShareIndex, sqrLn) + } + + return sid.RowID.Verify(root) +} + +// BlockFromFile returns the IPLD block of the Sample. +func (sid SampleID) BlockFromFile(ctx context.Context, f file.EdsFile) (blocks.Block, error) { + shr, err := f.Share(ctx, int(sid.ShareIndex), int(sid.RowID.RowIndex)) + if err != nil { + return nil, fmt.Errorf("while getting share with proof: %w", err) + } + + s := NewSample(sid, shr.Share, *shr.Proof, shr.Axis) + blk, err := s.IPLDBlock() + if err != nil { + return nil, fmt.Errorf("while coverting to IPLD block: %w", err) + } + return blk, nil +} + +// Release releases the verifier of the SampleID. +func (sid SampleID) Release() { + globalRootsCache.Delete(sid) +} + +func (sid SampleID) appendTo(data []byte) []byte { + data = sid.RowID.appendTo(data) + return binary.BigEndian.AppendUint16(data, sid.ShareIndex) +} diff --git a/share/shwap/sample_id_test.go b/share/shwap/sample_id_test.go new file mode 100644 index 0000000000..373cfa69fc --- /dev/null +++ b/share/shwap/sample_id_test.go @@ -0,0 +1,33 @@ +package shwap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/testing/edstest" +) + +func TestSampleID(t *testing.T) { + square := edstest.RandEDS(t, 2) + root, err := share.NewRoot(square) + require.NoError(t, err) + + id, err := NewSampleID(1, 1, root) + require.NoError(t, err) + + cid := id.Cid() + assert.EqualValues(t, sampleCodec, cid.Prefix().Codec) + assert.EqualValues(t, sampleMultihashCode, cid.Prefix().MhType) + assert.EqualValues(t, SampleIDSize, cid.Prefix().MhLength) + + data := id.MarshalBinary() + idOut, err := SampleIdFromBinary(data) + require.NoError(t, err) + assert.EqualValues(t, id, idOut) + + err = idOut.Verify(root) + require.NoError(t, err) +} diff --git a/share/shwap/sample_test.go b/share/shwap/sample_test.go new file mode 100644 index 0000000000..77acd310e0 --- /dev/null +++ b/share/shwap/sample_test.go @@ -0,0 +1,34 @@ +package shwap + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/testing/edstest" +) + +func TestSample(t *testing.T) { + square := edstest.RandEDS(t, 8) + root, err := share.NewRoot(square) + require.NoError(t, err) + + sample, err := NewSampleFromEDS(RowProofType, 1, square, 1) + require.NoError(t, err) + + data, err := sample.MarshalBinary() + require.NoError(t, err) + + blk, err := sample.IPLDBlock() + require.NoError(t, err) + assert.EqualValues(t, blk.Cid(), sample.Cid()) + + sampleOut, err := SampleFromBinary(data) + require.NoError(t, err) + assert.EqualValues(t, sample, sampleOut) + + err = sampleOut.Verify(root) + require.NoError(t, err) +} diff --git a/share/shwap/shwap.go b/share/shwap/shwap.go new file mode 100644 index 0000000000..53b80574c6 --- /dev/null +++ b/share/shwap/shwap.go @@ -0,0 +1,105 @@ +package shwap + +import ( + "crypto/sha256" + "fmt" + "hash" + "sync" + + "github.com/ipfs/go-cid" + logger "github.com/ipfs/go-log/v2" + mh "github.com/multiformats/go-multihash" + + "github.com/celestiaorg/celestia-node/share" +) + +var log = logger.Logger("shwap") + +const ( + // rowCodec is a CID codec used for row Bitswap requests over Namespaced Merkle + // Tree. + rowCodec = 0x7800 + + // rowMultihashCode is the multihash code for custom axis sampling multihash function. + rowMultihashCode = 0x7801 + + // sampleCodec is a CID codec used for share sampling Bitswap requests over Namespaced + // Merkle Tree. + sampleCodec = 0x7810 + + // sampleMultihashCode is the multihash code for share sampling multihash function. + sampleMultihashCode = 0x7811 + + // dataCodec is a CID codec used for data Bitswap requests over Namespaced Merkle Tree. + dataCodec = 0x7820 + + // dataMultihashCode is the multihash code for data multihash function. + dataMultihashCode = 0x7821 + + // mhPrefixSize is the size of the multihash prefix that used to cut it off. + mhPrefixSize = 4 +) + +var ( + hashFn = sha256.New +) + +func init() { + // Register hashers for new multihashes + mh.Register(rowMultihashCode, func() hash.Hash { + return &RowHasher{} + }) + mh.Register(sampleMultihashCode, func() hash.Hash { + return &SampleHasher{} + }) + mh.Register(dataMultihashCode, func() hash.Hash { + return &DataHasher{} + }) +} + +// TODO(@walldiss): store refscount along with roots to avoid verify errors on parallel requests +var globalRootsCache sync.Map + +func getRoot(key any) (*share.Root, error) { + r, ok := globalRootsCache.Load(key) + if !ok { + return nil, fmt.Errorf("no verifier") + } + + return r.(*share.Root), nil +} + +// DefaultAllowlist keeps default list of hashes allowed in the network. +var DefaultAllowlist allowlist + +type allowlist struct{} + +func (a allowlist) IsAllowed(code uint64) bool { + // we disable all codes except home-baked code + switch code { + case rowMultihashCode, sampleMultihashCode, dataMultihashCode: + return true + } + return false +} + +func validateCID(cid cid.Cid) error { + prefix := cid.Prefix() + if !DefaultAllowlist.IsAllowed(prefix.MhType) { + return fmt.Errorf("unsupported multihash type %d", prefix.MhType) + } + + switch prefix.Codec { + default: + return fmt.Errorf("unsupported codec %d", prefix.Codec) + case rowCodec, sampleCodec, dataCodec: + } + + switch prefix.MhLength { + default: + return fmt.Errorf("unsupported multihash length %d", prefix.MhLength) + case RowIDSize, SampleIDSize, DataIDSize: + } + + return nil +} diff --git a/share/shwap/shwap_test.go b/share/shwap/shwap_test.go new file mode 100644 index 0000000000..4610655d96 --- /dev/null +++ b/share/shwap/shwap_test.go @@ -0,0 +1,356 @@ +package shwap + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/ipfs/boxo/bitswap" + "github.com/ipfs/boxo/bitswap/network" + "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/exchange" + "github.com/ipfs/boxo/routing/offline" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + ipld "github.com/ipfs/go-ipld-format" + record "github.com/libp2p/go-libp2p-record" + mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/store/file" + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" +) + +// TestSampleRoundtripGetBlock tests full protocol round trip of: +// EDS -> Sample -> IPLDBlock -> BlockService -> Bitswap and in reverse. +func TestSampleRoundtripGetBlock(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + b := newTestBlockstore(t) + eds := edstest.RandEDS(t, 8) + height := b.AddEds(eds) + root, err := share.NewRoot(eds) + require.NoError(t, err) + + client := remoteClient(ctx, t, b) + + width := int(eds.Width()) + for i := 0; i < width*width; i++ { + smpl, err := NewSampleFromEDS(RowProofType, i, eds, height) // TODO: Col + require.NoError(t, err) + + globalRootsCache.Store(smpl.SampleID, root) + + cid := smpl.Cid() + blkOut, err := client.GetBlock(ctx, cid) + require.NoError(t, err) + require.EqualValues(t, cid, blkOut.Cid()) + + smpl, err = SampleFromBlock(blkOut) + require.NoError(t, err) + + err = smpl.Verify(root) + require.NoError(t, err) + } +} + +// TODO: Debug why is it flaky +func TestSampleRoundtripGetBlocks(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + b := newTestBlockstore(t) + eds := edstest.RandEDS(t, 8) + height := b.AddEds(eds) + root, err := share.NewRoot(eds) + require.NoError(t, err) + client := remoteClient(ctx, t, b) + + set := cid.NewSet() + width := int(eds.Width()) + for i := 0; i < width*width; i++ { + smpl, err := NewSampleFromEDS(RowProofType, i, eds, height) // TODO: Col + require.NoError(t, err) + set.Add(smpl.Cid()) + globalRootsCache.Store(smpl.SampleID, root) + } + + blks, err := client.GetBlocks(ctx, set.Keys()) + require.NoError(t, err) + + err = set.ForEach(func(c cid.Cid) error { + select { + case blk := <-blks: + require.True(t, set.Has(blk.Cid())) + + smpl, err := SampleFromBlock(blk) + require.NoError(t, err) + + err = smpl.Verify(root) // bitswap already performed validation and this is only for testing + require.NoError(t, err) + case <-ctx.Done(): + return ctx.Err() + } + return nil + }) + require.NoError(t, err) +} + +func TestRowRoundtripGetBlock(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + b := newTestBlockstore(t) + eds := edstest.RandEDS(t, 8) + height := b.AddEds(eds) + root, err := share.NewRoot(eds) + require.NoError(t, err) + client := remoteClient(ctx, t, b) + + width := int(eds.Width()) + for i := 0; i < width; i++ { + row, err := NewRowFromEDS(height, i, eds) + require.NoError(t, err) + + globalRootsCache.Store(row.RowID, root) + + cid := row.Cid() + blkOut, err := client.GetBlock(ctx, cid) + require.NoError(t, err) + require.EqualValues(t, cid, blkOut.Cid()) + + row, err = RowFromBlock(blkOut) + require.NoError(t, err) + + err = row.Verify(root) + require.NoError(t, err) + } +} + +func TestRowRoundtripGetBlocks(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + b := newTestBlockstore(t) + eds := edstest.RandEDS(t, 8) + height := b.AddEds(eds) + root, err := share.NewRoot(eds) + require.NoError(t, err) + client := remoteClient(ctx, t, b) + + set := cid.NewSet() + width := int(eds.Width()) + for i := 0; i < width; i++ { + row, err := NewRowFromEDS(height, i, eds) + require.NoError(t, err) + set.Add(row.Cid()) + globalRootsCache.Store(row.RowID, root) + } + + blks, err := client.GetBlocks(ctx, set.Keys()) + require.NoError(t, err) + + err = set.ForEach(func(c cid.Cid) error { + select { + case blk := <-blks: + require.True(t, set.Has(blk.Cid())) + + row, err := RowFromBlock(blk) + require.NoError(t, err) + + err = row.Verify(root) + require.NoError(t, err) + case <-ctx.Done(): + return ctx.Err() + } + return nil + }) + require.NoError(t, err) +} + +func TestDataRoundtripGetBlock(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + b := newTestBlockstore(t) + namespace := sharetest.RandV0Namespace() + eds, root := edstest.RandEDSWithNamespace(t, namespace, 64, 16) + height := b.AddEds(eds) + client := remoteClient(ctx, t, b) + + nds, err := NewDataFromEDS(eds, height, namespace) + require.NoError(t, err) + + for _, nd := range nds { + globalRootsCache.Store(nd.DataID, root) + + cid := nd.Cid() + blkOut, err := client.GetBlock(ctx, cid) + require.NoError(t, err) + require.EqualValues(t, cid, blkOut.Cid()) + + ndOut, err := DataFromBlock(blkOut) + require.NoError(t, err) + + err = ndOut.Verify(root) + require.NoError(t, err) + } +} + +func TestDataRoundtripGetBlocks(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + b := newTestBlockstore(t) + namespace := sharetest.RandV0Namespace() + eds, root := edstest.RandEDSWithNamespace(t, namespace, 64, 16) + height := b.AddEds(eds) + client := remoteClient(ctx, t, b) + + nds, err := NewDataFromEDS(eds, height, namespace) + require.NoError(t, err) + + set := cid.NewSet() + for _, nd := range nds { + set.Add(nd.Cid()) + globalRootsCache.Store(nd.DataID, root) + } + + blks, err := client.GetBlocks(ctx, set.Keys()) + require.NoError(t, err) + + err = set.ForEach(func(c cid.Cid) error { + select { + case blk := <-blks: + require.True(t, set.Has(blk.Cid())) + + smpl, err := DataFromBlock(blk) + require.NoError(t, err) + + err = smpl.Verify(root) + require.NoError(t, err) + case <-ctx.Done(): + return ctx.Err() + } + return nil + }) + require.NoError(t, err) +} + +func remoteClient(ctx context.Context, t *testing.T, bstore blockstore.Blockstore) exchange.Fetcher { + net, err := mocknet.FullMeshLinked(2) + require.NoError(t, err) + + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + routing := offline.NewOfflineRouter(dstore, record.NamespacedValidator{}) + _ = bitswap.New( + ctx, + network.NewFromIpfsHost(net.Hosts()[0], routing), + bstore, + ) + + dstoreClient := dssync.MutexWrap(ds.NewMapDatastore()) + bstoreClient := blockstore.NewBlockstore(dstoreClient) + routingClient := offline.NewOfflineRouter(dstoreClient, record.NamespacedValidator{}) + + bitswapClient := bitswap.New( + ctx, + network.NewFromIpfsHost(net.Hosts()[1], routingClient), + bstoreClient, + ) + + err = net.ConnectAllButSelf() + require.NoError(t, err) + + return bitswapClient +} + +type testBlockstore struct { + t *testing.T + lastHeight uint64 + blocks map[uint64]*file.MemFile +} + +func newTestBlockstore(t *testing.T) *testBlockstore { + return &testBlockstore{ + t: t, + lastHeight: 1, + blocks: make(map[uint64]*file.MemFile), + } +} + +func (t *testBlockstore) AddEds(eds *rsmt2d.ExtendedDataSquare) (height uint64) { + for { + if _, ok := t.blocks[t.lastHeight]; !ok { + break + } + t.lastHeight++ + } + t.blocks[t.lastHeight] = &file.MemFile{Eds: eds} + return t.lastHeight +} + +func (t *testBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error { + //TODO implement me + panic("not implemented") +} + +func (t *testBlockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) { + req, err := BlockBuilderFromCID(cid) + if err != nil { + return false, fmt.Errorf("while getting height from CID: %w", err) + } + + _, ok := t.blocks[req.GetHeight()] + return ok, nil +} + +func (t *testBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) { + req, err := BlockBuilderFromCID(cid) + if err != nil { + return nil, fmt.Errorf("while getting height from CID: %w", err) + } + + f, ok := t.blocks[req.GetHeight()] + if !ok { + return nil, ipld.ErrNotFound{Cid: cid} + } + return req.BlockFromFile(ctx, f) +} + +func (t *testBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { + req, err := BlockBuilderFromCID(cid) + if err != nil { + return 0, fmt.Errorf("while getting height from CID: %w", err) + } + + f, ok := t.blocks[req.GetHeight()] + if !ok { + return 0, ipld.ErrNotFound{Cid: cid} + } + return f.Size(), nil +} + +func (t *testBlockstore) Put(ctx context.Context, block blocks.Block) error { + panic("not implemented") +} + +func (t *testBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { + panic("not implemented") +} + +func (t *testBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + panic("not implemented") +} + +func (t *testBlockstore) HashOnRead(enabled bool) { + panic("not implemented") +} diff --git a/share/store/blockstore.go b/share/store/blockstore.go new file mode 100644 index 0000000000..7368df6ea5 --- /dev/null +++ b/share/store/blockstore.go @@ -0,0 +1,180 @@ +package store + +import ( + "context" + "errors" + "fmt" + + bstore "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/datastore/dshelp" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + ipld "github.com/ipfs/go-ipld-format" + + "github.com/celestiaorg/celestia-node/libs/utils" + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/share/store/cache" + "github.com/celestiaorg/celestia-node/share/store/file" +) + +//TODO(@walldiss): blockstore is now able to identify invalid cids(requests) by handling file.ErrOutOfBounds +// err. Ideally this case should lead to some penalty for the peer that sent the invalid request. The proper +// place for this logic is in the bitswap protocol, but it's not designed to handle such cases. It forces us +// to handle this case in the blockstore level. For now, we just log the error and return an error to the +// caller. We should revisit this issue and find a proper solution. + +var _ bstore.Blockstore = (*Blockstore)(nil) + +var ( + blockstoreCacheKey = datastore.NewKey("bs-cache") + errUnsupportedOperation = errors.New("unsupported operation") +) + +// Blockstore implements the bstore.Blockstore interface on an EDSStore. +// It is used to provide a custom blockstore interface implementation to achieve access to the +// underlying EDSStore. The main use-case is randomized sampling over the whole chain of EDS block +// data and getting data by namespace. +type Blockstore struct { + store *Store + ds datastore.Batching +} + +func NewBlockstore(store *Store, ds datastore.Batching) *Blockstore { + return &Blockstore{ + store: store, + ds: namespace.Wrap(ds, blockstoreCacheKey), + } +} + +func (bs *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) { + req, err := shwap.BlockBuilderFromCID(cid) + if err != nil { + return false, fmt.Errorf("get height from CID: %w", err) + } + + // check cache first + height := req.GetHeight() + _, err = bs.store.cache.Get(height) + if err == nil { + return true, nil + } + + has, err := bs.store.HasByHeight(ctx, height) + if err == nil { + return has, nil + } + if !errors.Is(err, ErrNotFound) { + return false, fmt.Errorf("has file: %w", err) + } + + // key wasn't found in top level blockstore, but could be in datastore while being reconstructed + dsHas, dsErr := bs.ds.Has(ctx, dshelp.MultihashToDsKey(cid.Hash())) + // TODO:(@walldoss): Only specific error should be treated as missing block, otherwise return error + if dsErr != nil { + return false, nil + } + return dsHas, nil +} + +func (bs *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) { + req, err := shwap.BlockBuilderFromCID(cid) + if err != nil { + return nil, fmt.Errorf("while getting height from CID: %w", err) + } + + height := req.GetHeight() + f, err := bs.store.cache.Second().GetOrLoad(ctx, height, bs.openFile(height)) + if err == nil { + defer utils.CloseAndLog(log, "file", f) + return req.BlockFromFile(ctx, f) + } + + if errors.Is(err, ErrNotFound) { + k := dshelp.MultihashToDsKey(cid.Hash()) + blockData, err := bs.ds.Get(ctx, k) + if err == nil { + return blocks.NewBlockWithCid(blockData, cid) + } + // nmt's GetNode expects an ipld.ErrNotFound when a cid is not found. + return nil, ipld.ErrNotFound{Cid: cid} + } + + log.Debugf("get blockstore for cid %s: %s", cid, err) + return nil, err +} + +func (bs *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { + // TODO(@Wondertan): There must be a way to derive size without reading, proving, serializing and + // allocating Sample's block.Block. + // NOTE:Bitswap uses GetSize also to determine if we have content stored or not + // so simply returning constant size is not an option + req, err := shwap.BlockBuilderFromCID(cid) + if err != nil { + return 0, fmt.Errorf("get height from CID: %w", err) + } + + height := req.GetHeight() + f, err := bs.store.cache.Second().GetOrLoad(ctx, height, bs.openFile(height)) + if err != nil { + return 0, fmt.Errorf("get file: %w", err) + } + defer utils.CloseAndLog(log, "file", f) + + return f.Size(), nil +} + +func (bs *Blockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error { + k := dshelp.MultihashToDsKey(cid.Hash()) + return bs.ds.Delete(ctx, k) +} + +func (bs *Blockstore) Put(ctx context.Context, blk blocks.Block) error { + k := dshelp.MultihashToDsKey(blk.Cid().Hash()) + // note: we leave duplicate resolution to the underlying datastore + return bs.ds.Put(ctx, k, blk.RawData()) +} + +func (bs *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { + if len(blocks) == 1 { + // performance fast-path + return bs.Put(ctx, blocks[0]) + } + + t, err := bs.ds.Batch(ctx) + if err != nil { + return err + } + for _, b := range blocks { + k := dshelp.MultihashToDsKey(b.Cid().Hash()) + err = t.Put(ctx, k, b.RawData()) + if err != nil { + return err + } + } + return t.Commit(ctx) +} + +// AllKeysChan is a noop on the EDS blockstore because the keys are not stored in a single CAR file. +func (bs *Blockstore) AllKeysChan(context.Context) (<-chan cid.Cid, error) { + err := fmt.Errorf("AllKeysChan is: %w", errUnsupportedOperation) + log.Warn(err) + return nil, err +} + +// HashOnRead is a noop on the EDS blockstore but an error cannot be returned due to the method +// signature from the blockstore interface. +func (bs *Blockstore) HashOnRead(bool) { + log.Warn("HashOnRead is a noop on the EDS blockstore") +} + +func (bs *Blockstore) openFile(height uint64) cache.OpenFileFn { + return func(ctx context.Context) (file.EdsFile, error) { + f, err := bs.store.getByHeight(height) + if err != nil { + return nil, fmt.Errorf("opening ODS file: %w", err) + } + return fileLoader(f)(ctx) + } +} diff --git a/share/store/blockstore_test.go b/share/store/blockstore_test.go new file mode 100644 index 0000000000..ab78445be4 --- /dev/null +++ b/share/store/blockstore_test.go @@ -0,0 +1,109 @@ +package store + +import ( + "context" + mrand "math/rand" + "testing" + "time" + + ds "github.com/ipfs/go-datastore" + ds_sync "github.com/ipfs/go-datastore/sync" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share/shwap" + "github.com/celestiaorg/celestia-node/share/store/cache" + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" +) + +//TODO: +// - add caching tests +// - add recontruction tests + +func TestBlockstoreGetShareSample(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + t.Cleanup(cancel) + + edsStore, err := NewStore(DefaultParameters(), t.TempDir()) + require.NoError(t, err) + + // disable cache + edsStore.cache = cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{}) + + height := uint64(100) + eds, dah := randomEDS(t) + + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + + bs := NewBlockstore(edsStore, ds_sync.MutexWrap(ds.NewMapDatastore())) + + t.Run("Sample", func(t *testing.T) { + width := int(eds.Width()) + for i := 0; i < width*width; i++ { + id, err := shwap.NewSampleID(height, i, dah) + require.NoError(t, err) + blk, err := bs.Get(ctx, id.Cid()) + require.NoError(t, err) + + sample, err := shwap.SampleFromBlock(blk) + require.NoError(t, err) + + err = sample.Verify(dah) + require.NoError(t, err) + require.EqualValues(t, id, sample.SampleID) + } + }) + + t.Run("Row", func(t *testing.T) { + width := int(eds.Width()) + for i := 0; i < width; i++ { + rowID, err := shwap.NewRowID(height, uint16(i), dah) + require.NoError(t, err) + + blk, err := bs.Get(ctx, rowID.Cid()) + require.NoError(t, err) + + row, err := shwap.RowFromBlock(blk) + require.NoError(t, err) + + err = row.Verify(dah) + require.NoError(t, err) + + require.EqualValues(t, rowID, row.RowID) + } + }) + + t.Run("NamespaceData", func(t *testing.T) { + size := 8 + namespace := sharetest.RandV0Namespace() + amount := mrand.Intn(size*size-1) + 1 + eds, dah := edstest.RandEDSWithNamespace(t, namespace, amount, size) + + height := uint64(42) + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + + for i, row := range dah.RowRoots { + if namespace.IsOutsideRange(row, row) { + continue + } + + dataID, err := shwap.NewDataID(height, uint16(i), namespace, dah) + require.NoError(t, err) + + blk, err := bs.Get(ctx, dataID.Cid()) + require.NoError(t, err) + + nd, err := shwap.DataFromBlock(blk) + require.NoError(t, err) + + err = nd.Verify(dah) + require.NoError(t, err) + + require.EqualValues(t, dataID, nd.DataID) + } + }) +} diff --git a/share/store/cache/accessor_cache.go b/share/store/cache/accessor_cache.go new file mode 100644 index 0000000000..c690074c23 --- /dev/null +++ b/share/store/cache/accessor_cache.go @@ -0,0 +1,232 @@ +package cache + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + lru "github.com/hashicorp/golang-lru/v2" + + "github.com/celestiaorg/celestia-node/share/store/file" +) + +const defaultCloseTimeout = time.Minute + +var _ Cache = (*FileCache)(nil) + +// FileCache implements the Cache interface using an LRU cache backend. +type FileCache struct { + // The name is a prefix that will be used for cache metrics if they are enabled. + name string + // stripedLocks prevents simultaneous RW access to the file cache for an accessor. Instead + // of using only one lock or one lock per key, we stripe the keys across 256 locks. 256 is + // chosen because it 0-255 is the range of values we get looking at the last byte of the key. + stripedLocks [256]*sync.RWMutex + // Caches the file for a given key for file read affinity, i.e., further reads will likely + // be from the same file. Maps (Datahash -> accessor). + cache *lru.Cache[key, *accessor] + + metrics *metrics +} + +// accessor is the value stored in Cache. It implements the file.EdsFile interface. It has a +// reference counted so that it can be removed from the cache only when all references are released. +type accessor struct { + file.EdsFile + + lock sync.Mutex + height uint64 + done chan struct{} + refs atomic.Int32 + isClosed bool +} + +func NewFileCache(name string, cacheSize int) (*FileCache, error) { + bc := &FileCache{ + name: name, + stripedLocks: [256]*sync.RWMutex{}, + } + + for i := range bc.stripedLocks { + bc.stripedLocks[i] = &sync.RWMutex{} + } + // Instantiate the file Cache. + bslru, err := lru.NewWithEvict[key, *accessor](cacheSize, bc.evictFn()) + if err != nil { + return nil, fmt.Errorf("failed to instantiate accessor cache: %w", err) + } + bc.cache = bslru + return bc, nil +} + +// evictFn will be invoked when an item is evicted from the cache. +func (bc *FileCache) evictFn() func(key, *accessor) { + return func(_ key, ac *accessor) { + // we can release accessor from cache early, while it is being closed in parallel routine + go func() { + err := ac.close() + if err != nil { + bc.metrics.observeEvicted(true) + log.Errorf("couldn't close accessor after cache eviction: %s", err) + return + } + bc.metrics.observeEvicted(false) + }() + } +} + +// Get retrieves the accessor for a given key from the Cache. If the Accessor is not in +// the Cache, it returns an ErrCacheMiss. +func (bc *FileCache) Get(key key) (file.EdsFile, error) { + lk := bc.getLock(key) + lk.RLock() + defer lk.RUnlock() + + ac, ok := bc.cache.Get(key) + if !ok { + bc.metrics.observeGet(false) + return nil, ErrCacheMiss + } + + bc.metrics.observeGet(true) + return newRefCloser(ac) +} + +// GetOrLoad attempts to get an item from the cache, and if not found, invokes +// the provided loader function to load it. +func (bc *FileCache) GetOrLoad(ctx context.Context, key key, loader OpenFileFn) (file.EdsFile, error) { + lk := bc.getLock(key) + lk.Lock() + defer lk.Unlock() + + ac, ok := bc.cache.Get(key) + if ok { + // return accessor, only if it is not closed yet + accessorWithRef, err := newRefCloser(ac) + if err == nil { + bc.metrics.observeGet(true) + return accessorWithRef, nil + } + } + + // accessor not found in cache or closed, so load new one using loader + f, err := loader(ctx) + if err != nil { + return nil, fmt.Errorf("unable to load accessor: %w", err) + } + + ac = &accessor{EdsFile: f} + // Create a new accessor first to increment the reference count in it, so it cannot get evicted + // from the inner lru cache before it is used. + rc, err := newRefCloser(ac) + if err != nil { + return nil, err + } + bc.cache.Add(key, ac) + return rc, nil +} + +// Remove removes the Accessor for a given key from the cache. +func (bc *FileCache) Remove(key key) error { + lk := bc.getLock(key) + lk.RLock() + ac, ok := bc.cache.Get(key) + lk.RUnlock() + if !ok { + // item is not in cache + return nil + } + if err := ac.close(); err != nil { + return err + } + // The cache will call evictFn on removal, where accessor close will be called. + bc.cache.Remove(key) + return nil +} + +// EnableMetrics enables metrics for the cache. +func (bc *FileCache) EnableMetrics() error { + var err error + bc.metrics, err = newMetrics(bc) + return err +} + +func (s *accessor) addRef() error { + s.lock.Lock() + defer s.lock.Unlock() + if s.isClosed { + // item is already closed and soon will be removed after all refs are released + return ErrCacheMiss + } + if s.refs.Add(1) == 1 { + // there were no refs previously and done channel was closed, reopen it by recreating + s.done = make(chan struct{}) + } + return nil +} + +func (s *accessor) removeRef() { + s.lock.Lock() + defer s.lock.Unlock() + if s.refs.Add(-1) <= 0 { + close(s.done) + } +} + +// close closes the accessor and removes it from the cache if it is not closed yet. It will block +// until all references are released or timeout is reached. +func (s *accessor) close() error { + s.lock.Lock() + if s.isClosed { + s.lock.Unlock() + // accessor will be closed by another goroutine + return nil + } + s.isClosed = true + done := s.done + s.lock.Unlock() + + select { + case <-done: + case <-time.After(defaultCloseTimeout): + return fmt.Errorf("closing file, some readers didn't close the file within timeout,"+ + " amount left: %v", s.refs.Load()) + } + if err := s.EdsFile.Close(); err != nil { + return fmt.Errorf("closing accessor: %w", err) + } + return nil +} + +// refCloser manages references to accessor from provided reader and removes the ref, when the +// Close is called +type refCloser struct { + *accessor + closeFn func() +} + +// newRefCloser creates new refCloser +func newRefCloser(abs *accessor) (*refCloser, error) { + if err := abs.addRef(); err != nil { + return nil, err + } + + var closeOnce sync.Once + return &refCloser{ + accessor: abs, + closeFn: func() { + closeOnce.Do(abs.removeRef) + }, + }, nil +} + +func (c *refCloser) Close() error { + c.closeFn() + return nil +} + +func (bc *FileCache) getLock(k key) *sync.RWMutex { + return bc.stripedLocks[byte(k%256)] +} diff --git a/share/eds/cache/accessor_cache_test.go b/share/store/cache/accessor_cache_test.go similarity index 93% rename from share/eds/cache/accessor_cache_test.go rename to share/store/cache/accessor_cache_test.go index 347b251a88..e910cc0017 100644 --- a/share/eds/cache/accessor_cache_test.go +++ b/share/store/cache/accessor_cache_test.go @@ -20,7 +20,7 @@ func TestAccessorCache(t *testing.T) { t.Run("add / get item from cache", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - cache, err := NewAccessorCache("test", 1) + cache, err := NewFileCache("test", 1) require.NoError(t, err) // add accessor to the cache @@ -48,7 +48,7 @@ func TestAccessorCache(t *testing.T) { t.Run("get blockstore from accessor", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - cache, err := NewAccessorCache("test", 1) + cache, err := NewFileCache("test", 1) require.NoError(t, err) // add accessor to the cache @@ -79,7 +79,7 @@ func TestAccessorCache(t *testing.T) { t.Run("remove an item", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - cache, err := NewAccessorCache("test", 1) + cache, err := NewFileCache("test", 1) require.NoError(t, err) // add accessor to the cache @@ -100,13 +100,13 @@ func TestAccessorCache(t *testing.T) { // check if item exists _, err = cache.Get(key) - require.ErrorIs(t, err, errCacheMiss) + require.ErrorIs(t, err, ErrCacheMiss) }) t.Run("successive reads should read the same data", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - cache, err := NewAccessorCache("test", 1) + cache, err := NewFileCache("test", 1) require.NoError(t, err) // add accessor to the cache @@ -133,7 +133,7 @@ func TestAccessorCache(t *testing.T) { t.Run("removed by eviction", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - cache, err := NewAccessorCache("test", 1) + cache, err := NewFileCache("test", 1) require.NoError(t, err) // add accessor to the cache @@ -160,13 +160,13 @@ func TestAccessorCache(t *testing.T) { // check if item evicted _, err = cache.Get(key) - require.ErrorIs(t, err, errCacheMiss) + require.ErrorIs(t, err, ErrCacheMiss) }) t.Run("close on accessor is not closing underlying accessor", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - cache, err := NewAccessorCache("test", 1) + cache, err := NewFileCache("test", 1) require.NoError(t, err) // add accessor to the cache @@ -193,7 +193,7 @@ func TestAccessorCache(t *testing.T) { t.Run("close on accessor should wait all readers to finish", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - cache, err := NewAccessorCache("test", 1) + cache, err := NewFileCache("test", 1) require.NoError(t, err) // add accessor to the cache @@ -226,9 +226,9 @@ func TestAccessorCache(t *testing.T) { require.NoError(t, err) mock.checkClosed(t, false) - // reads for item that is being evicted should result in errCacheMiss + // reads for item that is being evicted should result in ErrCacheMiss _, err = cache.Get(key) - require.ErrorIs(t, err, errCacheMiss) + require.ErrorIs(t, err, ErrCacheMiss) // close second reader and wait for accessor to be closed err = accessor2.Close() @@ -247,7 +247,7 @@ func TestAccessorCache(t *testing.T) { t.Run("slow reader should not block eviction", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - cache, err := NewAccessorCache("test", 1) + cache, err := NewFileCache("test", 1) require.NoError(t, err) // add accessor to the cache @@ -268,7 +268,7 @@ func TestAccessorCache(t *testing.T) { // first accessor should be evicted from cache _, err = cache.Get(key1) - require.ErrorIs(t, err, errCacheMiss) + require.ErrorIs(t, err, ErrCacheMiss) // first accessor should not be closed before all refs are released by Close() is calls. mock1.checkClosed(t, false) diff --git a/share/store/cache/cache.go b/share/store/cache/cache.go new file mode 100644 index 0000000000..e4f79d1e0d --- /dev/null +++ b/share/store/cache/cache.go @@ -0,0 +1,40 @@ +package cache + +import ( + "context" + "errors" + + logging "github.com/ipfs/go-log/v2" + "go.opentelemetry.io/otel" + + "github.com/celestiaorg/celestia-node/share/store/file" +) + +var ( + log = logging.Logger("share/eds/cache") + meter = otel.Meter("eds_store_cache") +) + +var ( + ErrCacheMiss = errors.New("accessor not found in blockstore cache") +) + +type OpenFileFn func(context.Context) (file.EdsFile, error) + +type key = uint64 + +// Cache is an interface that defines the basic Cache operations. +type Cache interface { + // Get returns the EDS file for the given key. + Get(key) (file.EdsFile, error) + + // GetOrLoad attempts to get an item from the Cache and, if not found, invokes + // the provided loader function to load it into the Cache. + GetOrLoad(context.Context, key, OpenFileFn) (file.EdsFile, error) + + // Remove removes an item from Cache. + Remove(key) error + + // EnableMetrics enables metrics in Cache + EnableMetrics() error +} diff --git a/share/eds/cache/doublecache.go b/share/store/cache/doublecache.go similarity index 80% rename from share/eds/cache/doublecache.go rename to share/store/cache/doublecache.go index a63eadee9e..567a189252 100644 --- a/share/eds/cache/doublecache.go +++ b/share/store/cache/doublecache.go @@ -3,7 +3,7 @@ package cache import ( "errors" - "github.com/filecoin-project/dagstore/shard" + "github.com/celestiaorg/celestia-node/share/store/file" ) // DoubleCache represents a Cache that looks into multiple caches one by one. @@ -20,16 +20,16 @@ func NewDoubleCache(first, second Cache) *DoubleCache { } // Get looks for an item in all the caches one by one and returns the Cache found item. -func (mc *DoubleCache) Get(key shard.Key) (Accessor, error) { - ac, err := mc.first.Get(key) +func (mc *DoubleCache) Get(key key) (file.EdsFile, error) { + accessor, err := mc.first.Get(key) if err == nil { - return ac, nil + return accessor, nil } return mc.second.Get(key) } // Remove removes an item from all underlying caches -func (mc *DoubleCache) Remove(key shard.Key) error { +func (mc *DoubleCache) Remove(key key) error { err1 := mc.first.Remove(key) err2 := mc.second.Remove(key) return errors.Join(err1, err2) diff --git a/share/eds/cache/metrics.go b/share/store/cache/metrics.go similarity index 96% rename from share/eds/cache/metrics.go rename to share/store/cache/metrics.go index 565a61a5e0..8d2943b277 100644 --- a/share/eds/cache/metrics.go +++ b/share/store/cache/metrics.go @@ -17,7 +17,7 @@ type metrics struct { evictedCounter metric.Int64Counter } -func newMetrics(bc *AccessorCache) (*metrics, error) { +func newMetrics(bc *FileCache) (*metrics, error) { metricsPrefix := "eds_blockstore_cache_" + bc.name evictedCounter, err := meter.Int64Counter(metricsPrefix+"_evicted_counter", diff --git a/share/store/cache/noop.go b/share/store/cache/noop.go new file mode 100644 index 0000000000..6d2906d5bb --- /dev/null +++ b/share/store/cache/noop.go @@ -0,0 +1,73 @@ +package cache + +import ( + "context" + "io" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/store/file" +) + +var _ Cache = (*NoopCache)(nil) + +// NoopCache implements noop version of Cache interface +type NoopCache struct{} + +func (n NoopCache) Get(key) (file.EdsFile, error) { + return nil, ErrCacheMiss +} + +func (n NoopCache) GetOrLoad(ctx context.Context, _ key, loader OpenFileFn) (file.EdsFile, error) { + return loader(ctx) +} + +func (n NoopCache) Remove(key) error { + return nil +} + +func (n NoopCache) EnableMetrics() error { + return nil +} + +var _ file.EdsFile = (*NoopFile)(nil) + +// NoopFile implements noop version of file.EdsFile interface +type NoopFile struct{} + +func (n NoopFile) Close() error { + return nil +} + +func (n NoopFile) Reader() (io.Reader, error) { + return nil, nil +} + +func (n NoopFile) Size() int { + return 0 +} + +func (n NoopFile) Height() uint64 { + return 0 +} + +func (n NoopFile) DataHash() share.DataHash { + return nil +} + +func (n NoopFile) Share(ctx context.Context, x, y int) (*share.ShareWithProof, error) { + return nil, nil +} + +func (n NoopFile) AxisHalf(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) (file.AxisHalf, error) { + return file.AxisHalf{}, nil +} + +func (n NoopFile) Data(ctx context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) { + return share.NamespacedRow{}, nil +} + +func (n NoopFile) EDS(ctx context.Context) (*rsmt2d.ExtendedDataSquare, error) { + return nil, nil +} diff --git a/share/store/file/axis_half.go b/share/store/file/axis_half.go new file mode 100644 index 0000000000..128899a3d4 --- /dev/null +++ b/share/store/file/axis_half.go @@ -0,0 +1,69 @@ +package file + +import ( + "fmt" + + "github.com/celestiaorg/celestia-node/share" +) + +type AxisHalf struct { + Shares []share.Share + IsParity bool +} + +func (a AxisHalf) Extended() ([]share.Share, error) { + if a.IsParity { + return reconstructShares(codec, a.Shares) + } + return extendShares(codec, a.Shares) +} + +func extendShares(codec Codec, original []share.Share) ([]share.Share, error) { + if len(original) == 0 { + return nil, fmt.Errorf("original shares are empty") + } + + sqLen := len(original) * 2 + shareSize := len(original[0]) + + enc, err := codec.Encoder(sqLen) + if err != nil { + return nil, fmt.Errorf("encoder: %w", err) + } + + shares := make([]share.Share, sqLen) + copy(shares, original) + for i := len(original); i < len(shares); i++ { + shares[i] = make([]byte, shareSize) + } + + err = enc.Encode(shares) + if err != nil { + return nil, fmt.Errorf("encoding: %w", err) + } + return shares, nil +} + +func reconstructShares(codec Codec, parity []share.Share) ([]share.Share, error) { + if len(parity) == 0 { + return nil, fmt.Errorf("parity shares are empty") + } + + sqLen := len(parity) * 2 + + enc, err := codec.Encoder(sqLen) + if err != nil { + return nil, fmt.Errorf("encoder: %w", err) + } + + shares := make([]share.Share, sqLen) + for i := sqLen / 2; i < sqLen; i++ { + shares[i] = parity[i-sqLen/2] + } + + err = enc.Reconstruct(shares) + if err != nil { + return nil, fmt.Errorf("reconstructing: %w", err) + } + return shares, nil +} diff --git a/share/store/file/axis_half_test.go b/share/store/file/axis_half_test.go new file mode 100644 index 0000000000..a96910ce79 --- /dev/null +++ b/share/store/file/axis_half_test.go @@ -0,0 +1,31 @@ +package file + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share/testing/sharetest" +) + +func TestExtendAxisHalf(t *testing.T) { + shares := sharetest.RandShares(t, 16) + + original := AxisHalf{ + Shares: shares, + IsParity: false, + } + + extended, err := original.Extended() + require.NoError(t, err) + + parity := AxisHalf{ + Shares: extended[len(shares):], + IsParity: true, + } + + parityExtended, err := parity.Extended() + require.NoError(t, err) + + require.Equal(t, extended, parityExtended) +} diff --git a/share/store/file/cache_file.go b/share/store/file/cache_file.go new file mode 100644 index 0000000000..189a16c500 --- /dev/null +++ b/share/store/file/cache_file.go @@ -0,0 +1,222 @@ +package file + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/ipfs/boxo/blockservice" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/ipld" +) + +var _ EdsFile = (*proofsCacheFile)(nil) + +type proofsCacheFile struct { + EdsFile + + // lock protects axisCache + lock sync.RWMutex + // axisCache caches the axis Shares and proofs + axisCache []map[int]inMemoryAxis + // disableCache disables caching of rows for testing purposes + disableCache bool +} + +type inMemoryAxis struct { + shares []share.Share + + // root will be set only when proofs are calculated + root []byte + proofs blockservice.BlockGetter +} + +func WithProofsCache(f EdsFile) EdsFile { + return &proofsCacheFile{ + EdsFile: f, + axisCache: []map[int]inMemoryAxis{make(map[int]inMemoryAxis), make(map[int]inMemoryAxis)}, + } +} + +func (f *proofsCacheFile) Share(ctx context.Context, x, y int) (*share.ShareWithProof, error) { + axisType, axisIdx, shrIdx := rsmt2d.Row, y, x + ax, err := f.axisWithProofs(ctx, axisType, axisIdx) + if err != nil { + return nil, err + } + + // build share proof from proofs cached for given axis + share, err := ipld.GetShareWithProof(ctx, ax.proofs, ax.root, shrIdx, f.Size(), axisType) + if err != nil { + return nil, fmt.Errorf("building proof from cache: %w", err) + } + + return share, nil +} + +func (f *proofsCacheFile) axisWithProofs(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) (inMemoryAxis, error) { + // return axis with proofs from cache if possible + ax, ok := f.getAxisFromCache(axisType, axisIdx) + if ax.proofs != nil { + return ax, nil + } + + // build proofs from Shares and cache them + if !ok { + shrs, err := f.axis(ctx, axisType, axisIdx) + if err != nil { + return inMemoryAxis{}, fmt.Errorf("get axis: %w", err) + } + ax.shares = shrs + } + + // calculate proofs + adder := ipld.NewProofsAdder(f.Size(), true) + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(f.Size()/2), uint(axisIdx), + nmt.NodeVisitor(adder.VisitFn())) + for _, shr := range ax.shares { + err := tree.Push(shr) + if err != nil { + return inMemoryAxis{}, fmt.Errorf("push Shares: %w", err) + } + } + + // build the tree + root, err := tree.Root() + if err != nil { + return inMemoryAxis{}, fmt.Errorf("calculating root: %w", err) + } + + ax.root = root + ax.proofs, err = newRowProofsGetter(adder.Proofs()) + if err != nil { + return inMemoryAxis{}, fmt.Errorf("creating proof getter: %w", err) + } + + if !f.disableCache { + f.storeAxisInCache(axisType, axisIdx, ax) + } + return ax, nil +} + +func (f *proofsCacheFile) AxisHalf(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) (AxisHalf, error) { + // return axis from cache if possible + ax, ok := f.getAxisFromCache(axisType, axisIdx) + if ok { + return AxisHalf{ + Shares: ax.shares[:f.Size()/2], + IsParity: false, + }, nil + } + + // read axis from file if axis is in the first quadrant + half, err := f.EdsFile.AxisHalf(ctx, axisType, axisIdx) + if err != nil { + return AxisHalf{}, fmt.Errorf("reading axis from inner file: %w", err) + } + + if !f.disableCache { + ax.shares, err = half.Extended() + if err != nil { + return AxisHalf{}, fmt.Errorf("extending Shares: %w", err) + } + f.storeAxisInCache(axisType, axisIdx, ax) + } + + return half, nil +} + +func (f *proofsCacheFile) Data(ctx context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) { + ax, err := f.axisWithProofs(ctx, rsmt2d.Row, rowIdx) + if err != nil { + return share.NamespacedRow{}, err + } + + row, proof, err := ipld.GetSharesByNamespace(ctx, ax.proofs, ax.root, namespace, f.Size()) + if err != nil { + return share.NamespacedRow{}, fmt.Errorf("Shares by namespace %s for row %v: %w", namespace.String(), rowIdx, err) + } + + return share.NamespacedRow{ + Shares: row, + Proof: proof, + }, nil +} + +func (f *proofsCacheFile) EDS(ctx context.Context) (*rsmt2d.ExtendedDataSquare, error) { + shares := make([][]byte, 0, f.Size()*f.Size()) + for i := 0; i < f.Size(); i++ { + ax, err := f.axis(ctx, rsmt2d.Row, i) + if err != nil { + return nil, err + } + shares = append(shares, ax...) + } + + eds, err := rsmt2d.ImportExtendedDataSquare( + shares, + share.DefaultRSMT2DCodec(), + wrapper.NewConstructor(uint64(f.Size())/2)) + if err != nil { + return nil, fmt.Errorf("recomputing data square: %w", err) + } + return eds, nil +} + +func (f *proofsCacheFile) axis(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) ([]share.Share, error) { + half, err := f.AxisHalf(ctx, axisType, axisIdx) + if err != nil { + return nil, err + } + + return half.Extended() +} + +func (f *proofsCacheFile) storeAxisInCache(axisType rsmt2d.Axis, axisIdx int, axis inMemoryAxis) { + f.lock.Lock() + defer f.lock.Unlock() + f.axisCache[axisType][axisIdx] = axis +} + +func (f *proofsCacheFile) getAxisFromCache(axisType rsmt2d.Axis, axisIdx int) (inMemoryAxis, bool) { + f.lock.RLock() + defer f.lock.RUnlock() + ax, ok := f.axisCache[axisType][axisIdx] + return ax, ok +} + +// rowProofsGetter implements blockservice.BlockGetter interface +type rowProofsGetter struct { + proofs map[cid.Cid]blocks.Block +} + +func newRowProofsGetter(rawProofs map[cid.Cid][]byte) (*rowProofsGetter, error) { + proofs := make(map[cid.Cid]blocks.Block, len(rawProofs)) + for k, v := range rawProofs { + b, err := blocks.NewBlockWithCid(v, k) + if err != nil { + return nil, err + } + proofs[k] = b + } + return &rowProofsGetter{proofs: proofs}, nil +} + +func (r rowProofsGetter) GetBlock(_ context.Context, c cid.Cid) (blocks.Block, error) { + if b, ok := r.proofs[c]; ok { + return b, nil + } + return nil, errors.New("block not found") +} + +func (r rowProofsGetter) GetBlocks(_ context.Context, _ []cid.Cid) <-chan blocks.Block { + panic("not implemented") +} diff --git a/share/store/file/cache_file_test.go b/share/store/file/cache_file_test.go new file mode 100644 index 0000000000..d4ae61a807 --- /dev/null +++ b/share/store/file/cache_file_test.go @@ -0,0 +1,39 @@ +package file + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/rsmt2d" +) + +func TestCacheFile(t *testing.T) { + size := 8 + newFile := func(eds *rsmt2d.ExtendedDataSquare) EdsFile { + path := t.TempDir() + "/testfile" + fl, err := CreateOdsFile(path, []byte{}, eds) + require.NoError(t, err) + return WithProofsCache(fl) + } + + t.Run("Share", func(t *testing.T) { + testFileShare(t, newFile, size) + }) + + t.Run("AxisHalf", func(t *testing.T) { + testFileAxisHalf(t, newFile, size) + }) + + t.Run("Data", func(t *testing.T) { + testFileData(t, newFile, size) + }) + + t.Run("EDS", func(t *testing.T) { + testFileEds(t, newFile, size) + }) + + t.Run("ReadOds", func(t *testing.T) { + testFileReader(t, newFile, size) + }) +} diff --git a/share/store/file/close_once_file.go b/share/store/file/close_once_file.go new file mode 100644 index 0000000000..b77e9b82ff --- /dev/null +++ b/share/store/file/close_once_file.go @@ -0,0 +1,84 @@ +package file + +import ( + "context" + "errors" + "io" + "sync/atomic" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" +) + +var _ EdsFile = (*closeOnceFile)(nil) + +var errFileClosed = errors.New("file closed") + +type closeOnceFile struct { + f EdsFile + size int + datahash share.DataHash + closed atomic.Bool +} + +func WithClosedOnce(f EdsFile) EdsFile { + return &closeOnceFile{ + f: f, + size: f.Size(), + datahash: f.DataHash(), + } +} + +func (c *closeOnceFile) Close() error { + if !c.closed.Swap(true) { + err := c.f.Close() + // release reference to the file to allow GC to collect all resources associated with it + c.f = nil + return err + } + return nil +} + +func (c *closeOnceFile) Reader() (io.Reader, error) { + if c.closed.Load() { + return nil, errFileClosed + } + return c.f.Reader() +} + +func (c *closeOnceFile) Size() int { + return c.size +} + +func (c *closeOnceFile) DataHash() share.DataHash { + return c.datahash +} + +func (c *closeOnceFile) Share(ctx context.Context, x, y int) (*share.ShareWithProof, error) { + if c.closed.Load() { + return nil, errFileClosed + } + return c.f.Share(ctx, x, y) +} + +func (c *closeOnceFile) AxisHalf(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) (AxisHalf, error) { + if c.closed.Load() { + return AxisHalf{}, errFileClosed + } + return c.f.AxisHalf(ctx, axisType, axisIdx) +} + +func (c *closeOnceFile) Data(ctx context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) { + if c.closed.Load() { + return share.NamespacedRow{}, errFileClosed + } + return c.f.Data(ctx, namespace, rowIdx) +} + +func (c *closeOnceFile) EDS(ctx context.Context) (*rsmt2d.ExtendedDataSquare, error) { + if c.closed.Load() { + return nil, errFileClosed + } + return c.f.EDS(ctx) +} diff --git a/share/store/file/codec.go b/share/store/file/codec.go new file mode 100644 index 0000000000..a27280be11 --- /dev/null +++ b/share/store/file/codec.go @@ -0,0 +1,38 @@ +package file + +import ( + "sync" + + "github.com/klauspost/reedsolomon" +) + +var codec Codec + +func init() { + codec = NewCodec() +} + +type Codec interface { + Encoder(len int) (reedsolomon.Encoder, error) +} + +type codecCache struct { + cache sync.Map +} + +func NewCodec() Codec { + return &codecCache{} +} + +func (l *codecCache) Encoder(len int) (reedsolomon.Encoder, error) { + enc, ok := l.cache.Load(len) + if !ok { + var err error + enc, err = reedsolomon.New(len/2, len/2, reedsolomon.WithLeopardGF(true)) + if err != nil { + return nil, err + } + l.cache.Store(len, enc) + } + return enc.(reedsolomon.Encoder), nil +} diff --git a/share/store/file/codec_test.go b/share/store/file/codec_test.go new file mode 100644 index 0000000000..f30f6ee72a --- /dev/null +++ b/share/store/file/codec_test.go @@ -0,0 +1,83 @@ +package file + +import ( + "fmt" + "testing" + + "github.com/klauspost/reedsolomon" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share/testing/sharetest" +) + +func BenchmarkCodec(b *testing.B) { + minSize, maxSize := 32, 128 + + for size := minSize; size <= maxSize; size *= 2 { + // BenchmarkCodec/Leopard/size:32-10 409194 2793 ns/op + // BenchmarkCodec/Leopard/size:64-10 190969 6170 ns/op + // BenchmarkCodec/Leopard/size:128-10 82821 14287 ns/op + b.Run(fmt.Sprintf("Leopard/size:%v", size), func(b *testing.B) { + enc, err := reedsolomon.New(size/2, size/2, reedsolomon.WithLeopardGF(true)) + require.NoError(b, err) + + shards := newShards(b, size, true) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err = enc.Encode(shards) + require.NoError(b, err) + } + }) + + // BenchmarkCodec/default/size:32-10 222153 5364 ns/op + // BenchmarkCodec/default/size:64-10 58831 20349 ns/op + // BenchmarkCodec/default/size:128-10 14940 80471 ns/op + b.Run(fmt.Sprintf("default/size:%v", size), func(b *testing.B) { + enc, err := reedsolomon.New(size/2, size/2, reedsolomon.WithLeopardGF(false)) + require.NoError(b, err) + + shards := newShards(b, size, true) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err = enc.Encode(shards) + require.NoError(b, err) + } + }) + + // BenchmarkCodec/default-reconstructSome/size:32-10 1263585 954.4 ns/op + // BenchmarkCodec/default-reconstructSome/size:64-10 762273 1554 ns/op + // BenchmarkCodec/default-reconstructSome/size:128-10 429268 2974 ns/op + b.Run(fmt.Sprintf("default-reconstructSome/size:%v", size), func(b *testing.B) { + enc, err := reedsolomon.New(size/2, size/2, reedsolomon.WithLeopardGF(false)) + require.NoError(b, err) + + shards := newShards(b, size, false) + targets := make([]bool, size) + target := size - 2 + targets[target] = true + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err = enc.ReconstructSome(shards, targets) + require.NoError(b, err) + shards[target] = nil + } + }) + } +} + +func newShards(b require.TestingT, size int, fillParity bool) [][]byte { + shards := make([][]byte, size) + original := sharetest.RandShares(b, size/2) + copy(shards, original) + + if fillParity { + // fill with parity empty Shares + for j := len(original); j < len(shards); j++ { + shards[j] = make([]byte, len(original[0])) + } + } + return shards +} diff --git a/share/store/file/file.go b/share/store/file/file.go new file mode 100644 index 0000000000..f77cee3d94 --- /dev/null +++ b/share/store/file/file.go @@ -0,0 +1,32 @@ +package file + +import ( + "context" + "io" + + logging "github.com/ipfs/go-log/v2" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" +) + +var log = logging.Logger("store/file") + +type EdsFile interface { + io.Closer + // Reader returns binary reader for the file. + Reader() (io.Reader, error) + // Size returns square size of the file. + Size() int + // DataHash returns data hash of the file. + DataHash() share.DataHash + // Share returns share and corresponding proof for the given axis and share index in this axis. + Share(ctx context.Context, x, y int) (*share.ShareWithProof, error) + // AxisHalf returns Shares for the first half of the axis of the given type and index. + AxisHalf(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) (AxisHalf, error) + // Data returns data for the given namespace and row index. + Data(ctx context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) + // EDS returns extended data square stored in the file. + EDS(ctx context.Context) (*rsmt2d.ExtendedDataSquare, error) +} diff --git a/share/store/file/file_header.go b/share/store/file/file_header.go new file mode 100644 index 0000000000..f025754bd7 --- /dev/null +++ b/share/store/file/file_header.go @@ -0,0 +1,83 @@ +package file + +import ( + "bytes" + "encoding/binary" + "io" + + "github.com/celestiaorg/celestia-node/share" +) + +const HeaderSize = 64 + +type Header struct { + version fileVersion + fileType fileType + + // Taken directly from EDS + shareSize uint16 + squareSize uint16 + + // TODO(@walldiss) store all heights in the header? + //heightÑ‹ []uint64 + datahash share.DataHash +} + +type fileVersion uint8 + +const ( + FileV0 fileVersion = iota +) + +type fileType uint8 + +const ( + ods fileType = iota + q1q4 +) + +func (h *Header) Version() fileVersion { + return h.version +} + +func (h *Header) ShareSize() int { + return int(h.shareSize) +} + +func (h *Header) SquareSize() int { + return int(h.squareSize) +} + +func (h *Header) DataHash() share.DataHash { + return h.datahash +} + +func (h *Header) WriteTo(w io.Writer) (int64, error) { + buf := make([]byte, HeaderSize) + buf[0] = byte(h.version) + buf[1] = byte(h.fileType) + binary.LittleEndian.PutUint16(buf[2:4], h.shareSize) + binary.LittleEndian.PutUint16(buf[4:6], h.squareSize) + copy(buf[32:64], h.datahash) + _, err := io.Copy(w, bytes.NewBuffer(buf)) + return HeaderSize, err +} + +func ReadHeader(r io.Reader) (*Header, error) { + buf := make([]byte, HeaderSize) + _, err := io.ReadFull(r, buf) + if err != nil { + return nil, err + } + + h := &Header{ + version: fileVersion(buf[0]), + fileType: fileType(buf[1]), + shareSize: binary.LittleEndian.Uint16(buf[2:4]), + squareSize: binary.LittleEndian.Uint16(buf[4:6]), + datahash: make([]byte, 32), + } + + copy(h.datahash, buf[32:64]) + return h, err +} diff --git a/share/store/file/file_test.go b/share/store/file/file_test.go new file mode 100644 index 0000000000..e61200ffbb --- /dev/null +++ b/share/store/file/file_test.go @@ -0,0 +1,258 @@ +package file + +import ( + "context" + "fmt" + mrand "math/rand" + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" +) + +type createFile func(eds *rsmt2d.ExtendedDataSquare) EdsFile + +func testFileShare(t *testing.T, createFile createFile, odsSize int) { + eds := edstest.RandEDS(t, odsSize) + fl := createFile(eds) + + dah, err := share.NewRoot(eds) + require.NoError(t, err) + + width := int(eds.Width()) + t.Run("single thread", func(t *testing.T) { + for x := 0; x < width; x++ { + for y := 0; y < width; y++ { + testShare(t, fl, eds, dah, x, y) + } + } + }) + + t.Run("parallel", func(t *testing.T) { + wg := sync.WaitGroup{} + for y := 0; y < width; y++ { + for x := 0; x < width; x++ { + wg.Add(1) + go func(x, y int) { + defer wg.Done() + testShare(t, fl, eds, dah, x, y) + }(x, y) + } + } + wg.Wait() + }) +} + +func testShare(t *testing.T, + fl EdsFile, + eds *rsmt2d.ExtendedDataSquare, + dah *share.Root, + x, y int) { + width := int(eds.Width()) + shr, err := fl.Share(context.TODO(), x, y) + require.NoError(t, err) + + var axishash []byte + if shr.Axis == rsmt2d.Row { + require.Equal(t, getAxis(eds, shr.Axis, y)[x], shr.Share) + axishash = dah.RowRoots[y] + } else { + require.Equal(t, getAxis(eds, shr.Axis, x)[y], shr.Share) + axishash = dah.ColumnRoots[x] + } + + ok := shr.Validate(axishash, x, y, width) + require.True(t, ok) +} + +func testFileData(t *testing.T, createFile createFile, size int) { + t.Run("included", func(t *testing.T) { + // generate EDS with random data and some Shares with the same namespace + namespace := sharetest.RandV0Namespace() + amount := mrand.Intn(size*size-1) + 1 + eds, dah := edstest.RandEDSWithNamespace(t, namespace, amount, size) + f := createFile(eds) + testData(t, f, namespace, dah) + }) + + t.Run("not included", func(t *testing.T) { + // generate EDS with random data and some Shares with the same namespace + eds := edstest.RandEDS(t, size) + dah, err := share.NewRoot(eds) + require.NoError(t, err) + + maxNs := nmt.MaxNamespace(dah.RowRoots[(len(dah.RowRoots))/2-1], share.NamespaceSize) + targetNs, err := share.Namespace(maxNs).AddInt(-1) + require.NoError(t, err) + + f := createFile(eds) + testData(t, f, targetNs, dah) + }) +} + +func testData(t *testing.T, f EdsFile, namespace share.Namespace, dah *share.Root) { + for i, root := range dah.RowRoots { + if !namespace.IsOutsideRange(root, root) { + nd, err := f.Data(context.Background(), namespace, i) + require.NoError(t, err) + ok := nd.Verify(root, namespace) + require.True(t, ok) + } + } +} + +func testFileAxisHalf(t *testing.T, createFile createFile, odsSize int) { + eds := edstest.RandEDS(t, odsSize) + fl := createFile(eds) + + t.Run("single thread", func(t *testing.T) { + for _, axisType := range []rsmt2d.Axis{rsmt2d.Col, rsmt2d.Row} { + for i := 0; i < int(eds.Width()); i++ { + half, err := fl.AxisHalf(context.Background(), axisType, i) + require.NoError(t, err) + require.Len(t, half.Shares, odsSize) + + var expected []share.Share + if half.IsParity { + expected = getAxis(eds, axisType, i)[odsSize:] + } else { + expected = getAxis(eds, axisType, i)[:odsSize] + } + + require.Equal(t, expected, half.Shares) + } + } + }) + + t.Run("parallel", func(t *testing.T) { + wg := sync.WaitGroup{} + for _, axisType := range []rsmt2d.Axis{rsmt2d.Col, rsmt2d.Row} { + for i := 0; i < int(eds.Width()); i++ { + wg.Add(1) + go func(axisType rsmt2d.Axis, idx int) { + defer wg.Done() + half, err := fl.AxisHalf(context.Background(), axisType, idx) + require.NoError(t, err) + require.Len(t, half.Shares, odsSize) + + var expected []share.Share + if half.IsParity { + expected = getAxis(eds, axisType, idx)[odsSize:] + } else { + expected = getAxis(eds, axisType, idx)[:odsSize] + } + + require.Equal(t, expected, half.Shares) + }(axisType, i) + } + } + wg.Wait() + }) +} + +func testFileEds(t *testing.T, createFile createFile, size int) { + eds := edstest.RandEDS(t, size) + fl := createFile(eds) + + eds2, err := fl.EDS(context.Background()) + require.NoError(t, err) + require.True(t, eds.Equals(eds2)) +} + +func testFileReader(t *testing.T, createFile createFile, odsSize int) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + eds := edstest.RandEDS(t, odsSize) + f := createFile(eds) + + // verify that the reader represented by file can be read from + // multiple times, without exhausting the underlying reader. + wg := sync.WaitGroup{} + for i := 0; i < 6; i++ { + wg.Add(1) + go func() { + defer wg.Done() + testReader(t, ctx, f, eds) + }() + } + wg.Wait() +} + +func testReader(t *testing.T, ctx context.Context, f EdsFile, eds *rsmt2d.ExtendedDataSquare) { + reader, err := f.Reader() + require.NoError(t, err) + + streamed, err := ReadEds(ctx, reader, f.Size()) + require.NoError(t, err) + require.True(t, eds.Equals(streamed)) +} + +func benchGetAxisFromFile(b *testing.B, newFile func(size int) EdsFile, minSize, maxSize int) { + for size := minSize; size <= maxSize; size *= 2 { + f := newFile(size) + + // loop over all possible axis types and quadrants + for _, axisType := range []rsmt2d.Axis{rsmt2d.Row, rsmt2d.Col} { + for _, squareHalf := range []int{0, 1} { + name := fmt.Sprintf("Size:%v/Axis:%s/squareHalf:%s", size, axisType, strconv.Itoa(squareHalf)) + b.Run(name, func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := f.AxisHalf(context.TODO(), axisType, f.Size()/2*(squareHalf)) + require.NoError(b, err) + } + }) + } + } + } +} + +func benchGetShareFromFile(b *testing.B, newFile func(size int) EdsFile, minSize, maxSize int) { + for size := minSize; size <= maxSize; size *= 2 { + f := newFile(size) + + // loop over all possible axis types and quadrants + for _, q := range quadrants { + name := fmt.Sprintf("Size:%v/quadrant:%s", size, q) + b.Run(name, func(b *testing.B) { + x, y := q.coordinates(f.Size()) + // warm up cache + _, err := f.Share(context.TODO(), x, y) + require.NoError(b, err) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := f.Share(context.TODO(), x, y) + require.NoError(b, err) + } + }) + } + + } +} + +type quadrant int + +var ( + quadrants = []quadrant{1, 2, 3, 4} +) + +func (q quadrant) String() string { + return strconv.Itoa(int(q)) +} + +func (q quadrant) coordinates(edsSize int) (x, y int) { + x = edsSize/2*(int(q-1)%2) + 1 + y = edsSize/2*(int(q-1)/2) + 1 + return +} diff --git a/share/store/file/mem_file.go b/share/store/file/mem_file.go new file mode 100644 index 0000000000..092aee9904 --- /dev/null +++ b/share/store/file/mem_file.go @@ -0,0 +1,136 @@ +package file + +import ( + "context" + "io" + + "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/ipld" +) + +var _ EdsFile = (*MemFile)(nil) + +type MemFile struct { + Eds *rsmt2d.ExtendedDataSquare +} + +func (f *MemFile) Close() error { + return nil +} + +func (f *MemFile) Reader() (io.Reader, error) { + return f.readOds().Reader() +} + +func (f *MemFile) readOds() square { + odsLn := int(f.Eds.Width() / 2) + s := make(square, odsLn) + for y := 0; y < odsLn; y++ { + s[y] = make([]share.Share, odsLn) + for x := 0; x < odsLn; x++ { + s[y][x] = f.Eds.GetCell(uint(y), uint(x)) + } + } + return s +} + +func (f *MemFile) DataHash() share.DataHash { + dah, _ := da.NewDataAvailabilityHeader(f.Eds) + return dah.Hash() +} + +func (f *MemFile) Size() int { + return int(f.Eds.Width()) +} + +func (f *MemFile) Share( + _ context.Context, + x, y int, +) (*share.ShareWithProof, error) { + axisType := rsmt2d.Row + axisIdx, shrIdx := y, x + + shares := getAxis(f.Eds, axisType, axisIdx) + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(f.Size()/2), uint(axisIdx)) + for _, shr := range shares { + err := tree.Push(shr) + if err != nil { + return nil, err + } + } + + proof, err := tree.ProveRange(shrIdx, shrIdx+1) + if err != nil { + return nil, err + } + + return &share.ShareWithProof{ + Share: shares[shrIdx], + Proof: &proof, + Axis: axisType, + }, nil +} + +func (f *MemFile) AxisHalf(_ context.Context, axisType rsmt2d.Axis, axisIdx int) (AxisHalf, error) { + return AxisHalf{ + Shares: getAxis(f.Eds, axisType, axisIdx)[:f.Size()/2], + IsParity: false, + }, nil +} + +func (f *MemFile) Data(_ context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) { + shares := getAxis(f.Eds, rsmt2d.Row, rowIdx) + return ndDataFromShares(shares, namespace, rowIdx) +} + +func (f *MemFile) EDS(_ context.Context) (*rsmt2d.ExtendedDataSquare, error) { + return f.Eds, nil +} + +func getAxis(eds *rsmt2d.ExtendedDataSquare, axisType rsmt2d.Axis, axisIdx int) []share.Share { + switch axisType { + case rsmt2d.Row: + return eds.Row(uint(axisIdx)) + case rsmt2d.Col: + return eds.Col(uint(axisIdx)) + default: + panic("unknown axis") + } +} + +func ndDataFromShares(shares []share.Share, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) { + bserv := ipld.NewMemBlockservice() + batchAdder := ipld.NewNmtNodeAdder(context.TODO(), bserv, ipld.MaxSizeBatchOption(len(shares))) + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(len(shares)/2), uint(rowIdx), + nmt.NodeVisitor(batchAdder.Visit)) + for _, shr := range shares { + err := tree.Push(shr) + if err != nil { + return share.NamespacedRow{}, err + } + } + + root, err := tree.Root() + if err != nil { + return share.NamespacedRow{}, err + } + + err = batchAdder.Commit() + if err != nil { + return share.NamespacedRow{}, err + } + + row, proof, err := ipld.GetSharesByNamespace(context.TODO(), bserv, root, namespace, len(shares)) + if err != nil { + return share.NamespacedRow{}, err + } + return share.NamespacedRow{ + Shares: row, + Proof: proof, + }, nil +} diff --git a/share/store/file/mem_file_test.go b/share/store/file/mem_file_test.go new file mode 100644 index 0000000000..f27c8ab8b1 --- /dev/null +++ b/share/store/file/mem_file_test.go @@ -0,0 +1,34 @@ +package file + +import ( + "testing" + + "github.com/celestiaorg/rsmt2d" +) + +func TestMemFile(t *testing.T) { + size := 8 + newFile := func(eds *rsmt2d.ExtendedDataSquare) EdsFile { + return &MemFile{Eds: eds} + } + + t.Run("Share", func(t *testing.T) { + testFileShare(t, newFile, size) + }) + + t.Run("AxisHalf", func(t *testing.T) { + testFileAxisHalf(t, newFile, size) + }) + + t.Run("Data", func(t *testing.T) { + testFileData(t, newFile, size) + }) + + t.Run("EDS", func(t *testing.T) { + testFileEds(t, newFile, size) + }) + + t.Run("ReadOds", func(t *testing.T) { + testFileReader(t, newFile, size) + }) +} diff --git a/share/store/file/mempool.go b/share/store/file/mempool.go new file mode 100644 index 0000000000..d290e0cdbc --- /dev/null +++ b/share/store/file/mempool.go @@ -0,0 +1,76 @@ +package file + +import ( + "sync" + + "github.com/celestiaorg/celestia-node/share" +) + +// TODO: need better name +var memPools poolsMap + +func init() { + memPools = make(map[int]*memPool) +} + +type poolsMap map[int]*memPool + +type memPool struct { + ods *sync.Pool + halfAxis *sync.Pool +} + +// TODO: test me +func (m poolsMap) get(size int) *memPool { + pool, ok := m[size] + if !ok { + pool = &memPool{ + ods: newOdsPool(size), + halfAxis: newHalfAxisPool(size), + } + m[size] = pool + } + return pool +} + +func (m *memPool) putSquare(s [][]share.Share) { + m.ods.Put(s) +} + +func (m *memPool) square() [][]share.Share { + return m.ods.Get().([][]share.Share) +} + +func (m *memPool) putHalfAxis(buf []byte) { + m.halfAxis.Put(buf) +} + +func (m *memPool) getHalfAxis() []byte { + return m.halfAxis.Get().([]byte) +} + +func newOdsPool(size int) *sync.Pool { + return &sync.Pool{ + New: func() interface{} { + shrs := make([][]share.Share, size) + for i := range shrs { + if shrs[i] == nil { + shrs[i] = make([]share.Share, size) + for j := range shrs[i] { + shrs[i][j] = make(share.Share, share.Size) + } + } + } + return shrs + }, + } +} + +func newHalfAxisPool(size int) *sync.Pool { + return &sync.Pool{ + New: func() interface{} { + buf := make([]byte, size*share.Size) + return buf + }, + } +} diff --git a/share/store/file/ods_file.go b/share/store/file/ods_file.go new file mode 100644 index 0000000000..e805856730 --- /dev/null +++ b/share/store/file/ods_file.go @@ -0,0 +1,279 @@ +package file + +import ( + "context" + "fmt" + "io" + "os" + "sync" + + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" +) + +var _ EdsFile = (*OdsFile)(nil) + +type OdsFile struct { + path string + hdr *Header + fl *os.File + + lock sync.RWMutex + ods square +} + +// OpenOdsFile opens an existing file. File has to be closed after usage. +func OpenOdsFile(path string) (*OdsFile, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + + h, err := ReadHeader(f) + if err != nil { + return nil, err + } + + return &OdsFile{ + path: path, + hdr: h, + fl: f, + }, nil +} + +func CreateOdsFile( + path string, + datahash share.DataHash, + eds *rsmt2d.ExtendedDataSquare) (*OdsFile, error) { + f, err := os.Create(path) + if err != nil { + return nil, fmt.Errorf("file create: %w", err) + } + + h := &Header{ + version: FileV0, + shareSize: share.Size, // TODO: rsmt2d should expose this field + squareSize: uint16(eds.Width()), + datahash: datahash, + } + + err = writeOdsFile(f, h, eds) + if err != nil { + return nil, fmt.Errorf("writing ODS file: %w", err) + } + + // TODO: fill ods field with data from eds + return &OdsFile{ + path: path, + fl: f, + hdr: h, + }, f.Sync() +} + +func writeOdsFile(w io.Writer, h *Header, eds *rsmt2d.ExtendedDataSquare) error { + _, err := h.WriteTo(w) + if err != nil { + return err + } + + for _, shr := range eds.FlattenedODS() { + if _, err := w.Write(shr); err != nil { + return err + } + } + return nil +} + +func (f *OdsFile) Size() int { + return f.hdr.SquareSize() +} + +func (f *OdsFile) Close() error { + if err := f.ods.close(); err != nil { + return err + } + return f.fl.Close() +} + +func (f *OdsFile) DataHash() share.DataHash { + return f.hdr.DataHash() +} + +func (f *OdsFile) Reader() (io.Reader, error) { + err := f.readOds() + if err != nil { + return nil, fmt.Errorf("reading ods: %w", err) + } + return f.ods.Reader() +} + +func (f *OdsFile) AxisHalf(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) (AxisHalf, error) { + // read axis from file if axis is in the first quadrant + if axisIdx < f.Size()/2 { + shares, err := f.readAxisHalf(axisType, axisIdx) + if err != nil { + return AxisHalf{}, fmt.Errorf("reading axis half: %w", err) + } + return AxisHalf{ + Shares: shares, + IsParity: false, + }, nil + } + + err := f.readOds() + if err != nil { + return AxisHalf{}, err + } + + shares, err := f.ods.computeAxisHalf(ctx, axisType, axisIdx) + if err != nil { + return AxisHalf{}, fmt.Errorf("computing axis half: %w", err) + } + return AxisHalf{ + Shares: shares, + IsParity: false, + }, nil +} + +func (f *OdsFile) readAxisHalf(axisType rsmt2d.Axis, axisIdx int) ([]share.Share, error) { + f.lock.RLock() + ods := f.ods + f.lock.RUnlock() + if ods != nil { + return f.ods.axisHalf(context.Background(), axisType, axisIdx) + } + + switch axisType { + case rsmt2d.Col: + return f.readCol(axisIdx, 0) + case rsmt2d.Row: + return f.readRow(axisIdx) + } + return nil, fmt.Errorf("unknown axis") +} + +func (f *OdsFile) readOds() error { + f.lock.Lock() + defer f.lock.Unlock() + if f.ods != nil { + return nil + } + + // reset file pointer to the beginning of the file + _, err := f.fl.Seek(HeaderSize, io.SeekStart) + if err != nil { + return fmt.Errorf("discarding header: %w", err) + } + + square, err := readSquare(f.fl, share.Size, f.Size()) + if err != nil { + return fmt.Errorf("reading ods: %w", err) + } + f.ods = square + return nil +} + +func (f *OdsFile) readRow(idx int) ([]share.Share, error) { + shrLn := int(f.hdr.shareSize) + odsLn := int(f.hdr.squareSize) / 2 + + shrs := make([]share.Share, odsLn) + + pos := idx * odsLn + offset := pos*shrLn + HeaderSize + + axsData := make([]byte, odsLn*shrLn) + if _, err := f.fl.ReadAt(axsData, int64(offset)); err != nil { + return nil, err + } + + for i := range shrs { + shrs[i] = axsData[i*shrLn : (i+1)*shrLn] + } + return shrs, nil +} + +func (f *OdsFile) readCol(axisIdx, quadrantIdx int) ([]share.Share, error) { + shrLn := int(f.hdr.shareSize) + odsLn := int(f.hdr.squareSize) / 2 + quadrantOffset := quadrantIdx * odsLn * odsLn * shrLn + + shrs := make([]share.Share, odsLn) + + for i := 0; i < odsLn; i++ { + pos := axisIdx + i*odsLn + offset := pos*shrLn + HeaderSize + quadrantOffset + + shr := make(share.Share, shrLn) + if _, err := f.fl.ReadAt(shr, int64(offset)); err != nil { + return nil, err + } + shrs[i] = shr + } + return shrs, nil +} + +func (f *OdsFile) Share(ctx context.Context, x, y int) (*share.ShareWithProof, error) { + axisType, axisIdx, shrIdx := rsmt2d.Row, y, x + // if the share is in the third quadrant, we need to switch axis type to column because it + // is more efficient to read single column than reading full ods to calculate single row + if x < f.Size()/2 && y >= f.Size()/2 { + axisType, axisIdx, shrIdx = rsmt2d.Col, x, y + } + + axis, err := f.axis(ctx, axisType, axisIdx) + if err != nil { + return nil, fmt.Errorf("reading axis: %w", err) + } + + return shareWithProof(axis, axisType, axisIdx, shrIdx) +} + +func (f *OdsFile) Data(ctx context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) { + shares, err := f.axis(ctx, rsmt2d.Row, rowIdx) + if err != nil { + return share.NamespacedRow{}, err + } + return ndDataFromShares(shares, namespace, rowIdx) +} + +func (f *OdsFile) EDS(_ context.Context) (*rsmt2d.ExtendedDataSquare, error) { + err := f.readOds() + if err != nil { + return nil, err + } + + return f.ods.eds() +} + +func shareWithProof(shares []share.Share, axisType rsmt2d.Axis, axisIdx, shrIdx int) (*share.ShareWithProof, error) { + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(len(shares)/2), uint(axisIdx)) + for _, shr := range shares { + err := tree.Push(shr) + if err != nil { + return nil, err + } + } + + proof, err := tree.ProveRange(shrIdx, shrIdx+1) + if err != nil { + return nil, err + } + + return &share.ShareWithProof{ + Share: shares[shrIdx], + Proof: &proof, + Axis: axisType, + }, nil +} + +func (f *OdsFile) axis(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) ([]share.Share, error) { + half, err := f.AxisHalf(ctx, axisType, axisIdx) + if err != nil { + return nil, err + } + + return half.Extended() +} diff --git a/share/store/file/ods_file_test.go b/share/store/file/ods_file_test.go new file mode 100644 index 0000000000..e9ef3f27ba --- /dev/null +++ b/share/store/file/ods_file_test.go @@ -0,0 +1,125 @@ +package file + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share/testing/edstest" +) + +func TestCreateOdsFile(t *testing.T) { + path := t.TempDir() + "/testfile" + edsIn := edstest.RandEDS(t, 8) + _, err := CreateOdsFile(path, []byte{}, edsIn) + require.NoError(t, err) + + f, err := OpenOdsFile(path) + require.NoError(t, err) + edsOut, err := f.EDS(context.TODO()) + require.NoError(t, err) + assert.True(t, edsIn.Equals(edsOut)) +} + +func TestReadFullOdsFromFile(t *testing.T) { + eds := edstest.RandEDS(t, 8) + path := t.TempDir() + "/testfile" + f, err := CreateOdsFile(path, []byte{}, eds) + require.NoError(t, err) + + err = f.readOds() + require.NoError(t, err) + for i, row := range f.ods { + original := eds.Row(uint(i))[:eds.Width()/2] + require.True(t, len(original) == len(row)) + require.Equal(t, original, row) + } +} + +func TestOdsFile(t *testing.T) { + size := 8 + createOdsFile := func(eds *rsmt2d.ExtendedDataSquare) EdsFile { + path := t.TempDir() + "/testfile" + fl, err := CreateOdsFile(path, []byte{}, eds) + require.NoError(t, err) + return fl + } + + t.Run("Share", func(t *testing.T) { + testFileShare(t, createOdsFile, size) + }) + + t.Run("AxisHalf", func(t *testing.T) { + testFileAxisHalf(t, createOdsFile, size) + }) + + t.Run("Data", func(t *testing.T) { + testFileData(t, createOdsFile, size) + }) + + t.Run("EDS", func(t *testing.T) { + testFileEds(t, createOdsFile, size) + }) + + t.Run("ReadOds", func(t *testing.T) { + testFileReader(t, createOdsFile, size) + }) +} + +// ReconstructSome, default codec +// BenchmarkAxisFromOdsFile/Size:32/Axis:row/squareHalf:first(original)-10 455848 2588 ns/op +// BenchmarkAxisFromOdsFile/Size:32/Axis:row/squareHalf:second(extended)-10 9015 203950 ns/op +// BenchmarkAxisFromOdsFile/Size:32/Axis:col/squareHalf:first(original)-10 52734 21178 ns/op +// BenchmarkAxisFromOdsFile/Size:32/Axis:col/squareHalf:second(extended)-10 8830 127452 ns/op +// BenchmarkAxisFromOdsFile/Size:64/Axis:row/squareHalf:first(original)-10 303834 4763 ns/op +// BenchmarkAxisFromOdsFile/Size:64/Axis:row/squareHalf:second(extended)-10 2940 426246 ns/op +// BenchmarkAxisFromOdsFile/Size:64/Axis:col/squareHalf:first(original)-10 27758 42842 ns/op +// BenchmarkAxisFromOdsFile/Size:64/Axis:col/squareHalf:second(extended)-10 3385 353868 ns/op +// BenchmarkAxisFromOdsFile/Size:128/Axis:row/squareHalf:first(original)-10 172086 6455 ns/op +// BenchmarkAxisFromOdsFile/Size:128/Axis:row/squareHalf:second(extended)-10 672 1550386 ns/op +// BenchmarkAxisFromOdsFile/Size:128/Axis:col/squareHalf:first(original)-10 14202 84316 ns/op +// BenchmarkAxisFromOdsFile/Size:128/Axis:col/squareHalf:second(extended)-10 978 1230980 ns/op +func BenchmarkAxisFromOdsFile(b *testing.B) { + minSize, maxSize := 32, 128 + dir := b.TempDir() + + newFile := func(size int) EdsFile { + eds := edstest.RandEDS(b, size) + path := dir + "/testfile" + f, err := CreateOdsFile(path, []byte{}, eds) + require.NoError(b, err) + return f + } + benchGetAxisFromFile(b, newFile, minSize, maxSize) +} + +// BenchmarkShareFromOdsFile/Size:32/Axis:row/squareHalf:first(original)-10 10339 111328 ns/op +// BenchmarkShareFromOdsFile/Size:32/Axis:row/squareHalf:second(extended)-10 3392 359180 ns/op +// BenchmarkShareFromOdsFile/Size:32/Axis:col/squareHalf:first(original)-10 8925 131352 ns/op +// BenchmarkShareFromOdsFile/Size:32/Axis:col/squareHalf:second(extended)-10 3447 346218 ns/op +// BenchmarkShareFromOdsFile/Size:64/Axis:row/squareHalf:first(original)-10 5503 215833 ns/op +// BenchmarkShareFromOdsFile/Size:64/Axis:row/squareHalf:second(extended)-10 1231 1001053 ns/op +// BenchmarkShareFromOdsFile/Size:64/Axis:col/squareHalf:first(original)-10 4711 250001 ns/op +// BenchmarkShareFromOdsFile/Size:64/Axis:col/squareHalf:second(extended)-10 1315 910079 ns/op +// BenchmarkShareFromOdsFile/Size:128/Axis:row/squareHalf:first(original)-10 2364 435748 ns/op +// BenchmarkShareFromOdsFile/Size:128/Axis:row/squareHalf:second(extended)-10 358 3330620 ns/op +// BenchmarkShareFromOdsFile/Size:128/Axis:col/squareHalf:first(original)-10 2114 514642 ns/op +// BenchmarkShareFromOdsFile/Size:128/Axis:col/squareHalf:second(extended)-10 373 3068104 ns/op +func BenchmarkShareFromOdsFile(b *testing.B) { + minSize, maxSize := 32, 128 + dir := b.TempDir() + + newFile := func(size int) EdsFile { + eds := edstest.RandEDS(b, size) + path := dir + "/testfile" + f, err := CreateOdsFile(path, []byte{}, eds) + require.NoError(b, err) + return f + } + + benchGetShareFromFile(b, newFile, minSize, maxSize) +} diff --git a/share/store/file/q1q4_file.go b/share/store/file/q1q4_file.go new file mode 100644 index 0000000000..78252a4706 --- /dev/null +++ b/share/store/file/q1q4_file.go @@ -0,0 +1,114 @@ +package file + +import ( + "context" + "fmt" + "io" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" +) + +var _ EdsFile = (*Q1Q4File)(nil) + +type Q1Q4File struct { + *OdsFile +} + +func OpenQ1Q4File(path string) (*Q1Q4File, error) { + ods, err := OpenOdsFile(path) + if err != nil { + return nil, err + } + + return &Q1Q4File{ + OdsFile: ods, + }, nil +} + +func CreateQ1Q4File( + path string, + datahash share.DataHash, + eds *rsmt2d.ExtendedDataSquare) (*Q1Q4File, error) { + ods, err := CreateOdsFile(path, datahash, eds) + if err != nil { + return nil, err + } + + err = writeQ4(ods.fl, eds) + if err != nil { + return nil, fmt.Errorf("writing Q4: %w", err) + } + + return &Q1Q4File{ + OdsFile: ods, + }, nil + +} + +func (f *Q1Q4File) AxisHalf(_ context.Context, axisType rsmt2d.Axis, axisIdx int) (AxisHalf, error) { + if axisIdx < f.Size()/2 { + half, err := f.OdsFile.readAxisHalf(axisType, axisIdx) + if err != nil { + return AxisHalf{}, fmt.Errorf("reading axis half: %w", err) + } + return AxisHalf{ + Shares: half, + IsParity: false, + }, nil + } + + var half []share.Share + var err error + switch axisType { + case rsmt2d.Col: + half, err = f.readCol(axisIdx-f.Size()/2, 1) + case rsmt2d.Row: + half, err = f.readRow(axisIdx) + } + if err != nil { + return AxisHalf{}, fmt.Errorf("reading axis: %w", err) + } + return AxisHalf{ + Shares: half, + IsParity: true, + }, nil +} + +func (f *Q1Q4File) Share(ctx context.Context, x, y int) (*share.ShareWithProof, error) { + half, err := f.AxisHalf(ctx, rsmt2d.Row, y) + if err != nil { + return nil, fmt.Errorf("reading axis: %w", err) + } + shares, err := half.Extended() + if err != nil { + return nil, fmt.Errorf("extending shares: %w", err) + } + return shareWithProof(shares, rsmt2d.Row, y, x) +} + +func (f *Q1Q4File) Data(ctx context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) { + half, err := f.AxisHalf(ctx, rsmt2d.Row, rowIdx) + if err != nil { + return share.NamespacedRow{}, fmt.Errorf("reading axis: %w", err) + } + shares, err := half.Extended() + if err != nil { + return share.NamespacedRow{}, fmt.Errorf("extending shares: %w", err) + } + return ndDataFromShares(shares, namespace, rowIdx) +} + +func writeQ4(w io.Writer, eds *rsmt2d.ExtendedDataSquare) error { + odsLn := int(eds.Width()) / 2 + for x := odsLn; x < int(eds.Width()); x++ { + for y := odsLn; y < int(eds.Width()); y++ { + _, err := w.Write(eds.GetCell(uint(x), uint(y))) + if err != nil { + return err + } + } + } + return nil +} diff --git a/share/store/file/q1q4_file_test.go b/share/store/file/q1q4_file_test.go new file mode 100644 index 0000000000..f488d4d2b1 --- /dev/null +++ b/share/store/file/q1q4_file_test.go @@ -0,0 +1,39 @@ +package file + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/rsmt2d" +) + +func TestQ1Q4File(t *testing.T) { + size := 8 + createOdsFile := func(eds *rsmt2d.ExtendedDataSquare) EdsFile { + path := t.TempDir() + "/testfile" + fl, err := CreateQ1Q4File(path, []byte{}, eds) + require.NoError(t, err) + return fl + } + + t.Run("Share", func(t *testing.T) { + testFileShare(t, createOdsFile, size) + }) + + t.Run("AxisHalf", func(t *testing.T) { + testFileAxisHalf(t, createOdsFile, size) + }) + + t.Run("Data", func(t *testing.T) { + testFileData(t, createOdsFile, size) + }) + + t.Run("EDS", func(t *testing.T) { + testFileEds(t, createOdsFile, size) + }) + + t.Run("ReadOds", func(t *testing.T) { + testFileReader(t, createOdsFile, size) + }) +} diff --git a/share/store/file/square.go b/share/store/file/square.go new file mode 100644 index 0000000000..e084d53af0 --- /dev/null +++ b/share/store/file/square.go @@ -0,0 +1,198 @@ +package file + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + + "golang.org/x/sync/errgroup" + + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" +) + +type square [][]share.Share + +// ReadEds reads an EDS from the reader and returns it. +func ReadEds(_ context.Context, r io.Reader, edsSize int) (*rsmt2d.ExtendedDataSquare, error) { + square, err := readSquare(r, share.Size, edsSize) + if err != nil { + return nil, fmt.Errorf("reading Shares: %w", err) + } + + eds, err := square.eds() + if err != nil { + return nil, fmt.Errorf("computing EDS: %w", err) + } + return eds, nil +} + +// readSquare reads Shares from the reader and returns a square. It assumes that the reader is +// positioned at the beginning of the Shares. It knows the size of the Shares and the size of the +// square, so reads from reader are limited to exactly the amount of data required. +func readSquare(r io.Reader, shareSize, edsSize int) (square, error) { + odsLn := edsSize / 2 + + // get pre-allocated square and buffer from memPools + square := memPools.get(odsLn).square() + + // TODO(@walldiss): run benchmark to find optimal size for buffer + br := bufio.NewReaderSize(r, 4096) + var total int + for i := 0; i < odsLn; i++ { + for j := 0; j < odsLn; j++ { + n, err := io.ReadFull(br, square[i][j]) + if err != nil { + return nil, fmt.Errorf("reading share: %w, bytes read: %v", err, total+n) + } + if n != shareSize { + return nil, fmt.Errorf("share size mismatch: expected %v, got %v", shareSize, n) + } + total += n + } + } + return square, nil +} + +func (s square) size() int { + return len(s) +} + +func (s square) close() error { + if s != nil { + // return square to memPools + memPools.get(s.size()).putSquare(s) + } + return nil +} + +func (s square) axisHalf(_ context.Context, axisType rsmt2d.Axis, axisIdx int) ([]share.Share, error) { + if s == nil { + return nil, fmt.Errorf("square is nil") + } + + if axisIdx >= s.size() { + return nil, fmt.Errorf("index is out of square bounds") + } + + // square stores rows directly in high level slice, so we can return by accessing row by index + if axisType == rsmt2d.Row { + return s[axisIdx], nil + } + + // construct half column from row ordered square + col := make([]share.Share, s.size()) + for i := 0; i < s.size(); i++ { + col[i] = s[i][axisIdx] + } + return col, nil +} + +func (s square) eds() (*rsmt2d.ExtendedDataSquare, error) { + //TODO(@walldiss): use mempool + shrs := make([]share.Share, 0, 4*s.size()*s.size()) + for _, row := range s { + shrs = append(shrs, row...) + } + + treeFn := wrapper.NewConstructor(uint64(s.size())) + return rsmt2d.ComputeExtendedDataSquare(shrs, share.DefaultRSMT2DCodec(), treeFn) +} + +func (s square) Reader() (io.Reader, error) { + if s == nil { + return nil, fmt.Errorf("ods file not cached") + } + + odsR := &bufferedODSReader{ + square: s, + total: s.size() * s.size(), + buf: new(bytes.Buffer), + } + + return odsR, nil +} + +func (s square) computeAxisHalf( + ctx context.Context, + axisType rsmt2d.Axis, + axisIdx int, +) ([]share.Share, error) { + shares := make([]share.Share, s.size()) + + // extend opposite half of the square while collecting Shares for the first half of required axis + g, ctx := errgroup.WithContext(ctx) + opposite := oppositeAxis(axisType) + for i := 0; i < s.size(); i++ { + i := i + g.Go(func() error { + original, err := s.axisHalf(ctx, opposite, i) + if err != nil { + return err + } + + enc, err := codec.Encoder(s.size() * 2) + if err != nil { + return fmt.Errorf("encoder: %w", err) + } + + shards := make([][]byte, s.size()*2) + copy(shards, original) + //for j := len(original); j < len(shards); j++ { + // shards[j] = make([]byte, len(original[0])) + //} + + //err = enc.Encode(shards) + //if err != nil { + // return fmt.Errorf("encode: %w", err) + //} + + target := make([]bool, s.size()*2) + target[axisIdx] = true + + err = enc.ReconstructSome(shards, target) + if err != nil { + return fmt.Errorf("reconstruct some: %w", err) + } + + shares[i] = shards[axisIdx] + return nil + }) + } + + err := g.Wait() + return shares, err +} + +func oppositeAxis(axis rsmt2d.Axis) rsmt2d.Axis { + if axis == rsmt2d.Col { + return rsmt2d.Row + } + return rsmt2d.Col +} + +// bufferedODSReader will read Shares from inMemOds into the buffer. +// It exposes the buffer to be read by io.Reader interface implementation +type bufferedODSReader struct { + square square + // current is the amount of Shares stored in square that have been read from reader. When current + // reaches total, bufferedODSReader will prevent further reads by returning io.EOF + current, total int + buf *bytes.Buffer +} + +func (r *bufferedODSReader) Read(p []byte) (n int, err error) { + // read Shares to the buffer until it has sufficient data to fill provided container or full square is + // read + for r.current < r.total && r.buf.Len() < len(p) { + x, y := r.current%(r.square.size()), r.current/(r.square.size()) + r.buf.Write(r.square[y][x]) + r.current++ + } + // read buffer to slice + return r.buf.Read(p) +} diff --git a/share/store/file/validating_file.go b/share/store/file/validating_file.go new file mode 100644 index 0000000000..ba36bc79e3 --- /dev/null +++ b/share/store/file/validating_file.go @@ -0,0 +1,55 @@ +package file + +import ( + "context" + "errors" + "fmt" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" +) + +// ErrOutOfBounds is returned whenever an index is out of bounds. +var ErrOutOfBounds = errors.New("index is out of bounds") + +// validatingFile is a file implementation that performs sanity checks on file operations. +type validatingFile struct { + EdsFile +} + +func WithValidation(f EdsFile) EdsFile { + return &validatingFile{EdsFile: f} +} + +func (f *validatingFile) Share(ctx context.Context, x, y int) (*share.ShareWithProof, error) { + if err := validateIndexBounds(f, x); err != nil { + return nil, fmt.Errorf("col: %w", err) + } + if err := validateIndexBounds(f, y); err != nil { + return nil, fmt.Errorf("row: %w", err) + } + return f.EdsFile.Share(ctx, x, y) +} + +func (f *validatingFile) AxisHalf(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) (AxisHalf, error) { + if err := validateIndexBounds(f, axisIdx); err != nil { + return AxisHalf{}, fmt.Errorf("%s: %w", axisType, err) + } + return f.EdsFile.AxisHalf(ctx, axisType, axisIdx) +} + +func (f *validatingFile) Data(ctx context.Context, namespace share.Namespace, rowIdx int) (share.NamespacedRow, error) { + if err := validateIndexBounds(f, rowIdx); err != nil { + return share.NamespacedRow{}, fmt.Errorf("row: %w", err) + } + return f.EdsFile.Data(ctx, namespace, rowIdx) +} + +// validateIndexBounds checks if the index is within the bounds of the file. +func validateIndexBounds(f EdsFile, idx int) error { + if idx < 0 || idx >= f.Size() { + return fmt.Errorf("%w: index %d is out of bounds: [0, %d)", ErrOutOfBounds, idx, f.Size()) + } + return nil +} diff --git a/share/store/file/validating_file_test.go b/share/store/file/validating_file_test.go new file mode 100644 index 0000000000..748a2d88e2 --- /dev/null +++ b/share/store/file/validating_file_test.go @@ -0,0 +1,103 @@ +package file + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/testing/edstest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" +) + +func TestValidatingFile_Share(t *testing.T) { + tests := []struct { + name string + x, y int + odsSize int + expectFail bool + }{ + {"ValidIndices", 3, 2, 4, false}, + {"OutOfBoundsX", 8, 3, 4, true}, + {"OutOfBoundsY", 3, 8, 4, true}, + {"NegativeX", -1, 4, 8, true}, + {"NegativeY", 3, -1, 8, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eds := edstest.RandEDS(t, tt.odsSize) + file := &MemFile{Eds: eds} + vf := WithValidation(file) + + _, err := vf.Share(context.Background(), tt.x, tt.y) + if tt.expectFail { + require.ErrorIs(t, err, ErrOutOfBounds) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestValidatingFile_AxisHalf(t *testing.T) { + tests := []struct { + name string + axisType rsmt2d.Axis + axisIdx int + odsSize int + expectFail bool + }{ + {"ValidIndex", rsmt2d.Row, 2, 4, false}, + {"OutOfBounds", rsmt2d.Col, 8, 4, true}, + {"NegativeIndex", rsmt2d.Row, -1, 4, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eds := edstest.RandEDS(t, tt.odsSize) + file := &MemFile{Eds: eds} + vf := WithValidation(file) + + _, err := vf.AxisHalf(context.Background(), tt.axisType, tt.axisIdx) + if tt.expectFail { + require.ErrorIs(t, err, ErrOutOfBounds) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestValidatingFile_Data(t *testing.T) { + tests := []struct { + name string + rowIdx int + odsSize int + expectFail bool + }{ + {"ValidIndex", 3, 4, false}, + {"OutOfBounds", 8, 4, true}, + {"NegativeIndex", -1, 4, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eds := edstest.RandEDS(t, tt.odsSize) + file := &MemFile{Eds: eds} + vf := WithValidation(file) + + ns := sharetest.RandV0Namespace() + _, err := vf.Data(context.Background(), ns, tt.rowIdx) + if tt.expectFail { + require.ErrorIs(t, err, ErrOutOfBounds) + } else { + require.True(t, err == nil || errors.Is(err, ipld.ErrNamespaceOutsideRange)) + } + }) + } +} diff --git a/share/store/metrics.go b/share/store/metrics.go new file mode 100644 index 0000000000..b251715be9 --- /dev/null +++ b/share/store/metrics.go @@ -0,0 +1,132 @@ +package store + +import ( + "context" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +const ( + failedKey = "failed" + sizeKey = "eds_size" +) + +var ( + meter = otel.Meter("store") +) + +type metrics struct { + put metric.Float64Histogram + putExists metric.Int64Counter + get metric.Float64Histogram + has metric.Float64Histogram + remove metric.Float64Histogram +} + +func (s *Store) WithMetrics() error { + put, err := meter.Float64Histogram("eds_store_put_time_histogram", + metric.WithDescription("eds store put time histogram(s)")) + if err != nil { + return err + } + + putExists, err := meter.Int64Counter("eds_store_put_exists_counter", + metric.WithDescription("eds store put file exists")) + if err != nil { + return err + } + + get, err := meter.Float64Histogram("eds_store_get_time_histogram", + metric.WithDescription("eds store get time histogram(s)")) + if err != nil { + return err + } + + has, err := meter.Float64Histogram("eds_store_has_time_histogram", + metric.WithDescription("eds store has time histogram(s)")) + if err != nil { + return err + } + + remove, err := meter.Float64Histogram("eds_store_remove_time_histogram", + metric.WithDescription("eds store remove time histogram(s)")) + if err != nil { + return err + } + + if err = s.cache.EnableMetrics(); err != nil { + return err + } + + s.metrics = &metrics{ + put: put, + putExists: putExists, + get: get, + has: has, + remove: remove, + } + return nil +} + +func (m *metrics) observePut(ctx context.Context, dur time.Duration, size uint, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.put.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed), + attribute.Int(sizeKey, int(size)))) +} + +func (m *metrics) observePutExist(ctx context.Context) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.putExists.Add(ctx, 1) +} + +func (m *metrics) observeGet(ctx context.Context, dur time.Duration, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.get.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} + +func (m *metrics) observeHas(ctx context.Context, dur time.Duration, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.has.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} + +func (m *metrics) observeRemove(ctx context.Context, dur time.Duration, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.remove.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} diff --git a/share/store/store.go b/share/store/store.go new file mode 100644 index 0000000000..8262109095 --- /dev/null +++ b/share/store/store.go @@ -0,0 +1,504 @@ +package store + +import ( + "context" + "encoding/gob" + "errors" + "fmt" + "io" + "os" + "strconv" + "sync" + "syscall" + "time" + + logging "github.com/ipfs/go-log/v2" + "go.opentelemetry.io/otel" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/libs/utils" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/store/cache" + "github.com/celestiaorg/celestia-node/share/store/file" +) + +var ( + log = logging.Logger("share/eds") + tracer = otel.Tracer("share/eds") + + emptyFile = &file.MemFile{Eds: share.EmptyExtendedDataSquare()} +) + +// TODO(@walldiss): +// - periodically persist empty heights +// - persist store stats like +// - amount of files +// - file types hist (ods/q1q4) +// - file size hist +// - amount of links hist +// - add handling of corrupted files / links +// - maintain in-memory missing files index / bloom-filter to fast return for not stored files. +// - lock store folder +// - add traces + +const ( + blocksPath = "/blocks/" + heightsPath = blocksPath + "heights/" + emptyHeightsFile = heightsPath + "empty_heights" + + defaultDirPerm = 0755 +) + +var ErrNotFound = errors.New("eds not found in store") + +// Store maintains (via DAGStore) a top-level index enabling granular and efficient random access to +// every share and/or Merkle proof over every registered CARv1 file. The EDSStore provides a custom +// blockstore interface implementation to achieve access. The main use-case is randomized sampling +// over the whole chain of EDS block data and getting data by namespace. +type Store struct { + // basepath is the root directory of the store + basepath string + // cache is used to cache recent blocks and blocks that are accessed frequently + cache *cache.DoubleCache + // stripedLocks is used to synchronize parallel operations + stripLock *striplock + // emptyHeights stores the heights of empty files + emptyHeights map[uint64]struct{} + emptyHeightsLock sync.RWMutex + + metrics *metrics +} + +// NewStore creates a new EDS Store under the given basepath and datastore. +func NewStore(params *Parameters, basePath string) (*Store, error) { + if err := params.Validate(); err != nil { + return nil, err + } + + // Ensure the blocks folder exists or is created. + blocksFolderPath := basePath + blocksPath + if err := ensureFolder(blocksFolderPath); err != nil { + log.Errorf("Failed to ensure the existence of the blocks folder at '%s': %s", blocksFolderPath, err) + return nil, fmt.Errorf("ensure blocks folder '%s': %w", blocksFolderPath, err) + } + + // Ensure the heights folder exists or is created. + heightsFolderPath := basePath + heightsPath + if err := ensureFolder(heightsFolderPath); err != nil { + log.Errorf("Failed to ensure the existence of the heights folder at '%s': %s", heightsFolderPath, err) + return nil, fmt.Errorf("ensure heights folder '%s': %w", heightsFolderPath, err) + } + + // Ensure the empty heights file exists or is created. + emptyHeightsFilePath := basePath + emptyHeightsFile + if err := ensureFile(emptyHeightsFilePath); err != nil { + log.Errorf("Failed to ensure the empty heights file at '%s': %s", emptyHeightsFilePath, err) + return nil, fmt.Errorf("ensure empty heights file '%s': %w", emptyHeightsFilePath, err) + } + + recentBlocksCache, err := cache.NewFileCache("recent", 1) + if err != nil { + return nil, fmt.Errorf("failed to create recent blocks cache: %w", err) + } + + blockstoreCache, err := cache.NewFileCache("blockstore", 1) + if err != nil { + return nil, fmt.Errorf("failed to create blockstore cache: %w", err) + } + + emptyHeights, err := loadEmptyHeights(basePath) + if err != nil { + return nil, fmt.Errorf("loading empty heights: %w", err) + } + + store := &Store{ + basepath: basePath, + cache: cache.NewDoubleCache(recentBlocksCache, blockstoreCache), + stripLock: newStripLock(1024), + emptyHeights: emptyHeights, + } + return store, nil +} + +func (s *Store) Close() error { + return s.storeEmptyHeights() +} + +func (s *Store) Put( + ctx context.Context, + datahash share.DataHash, + height uint64, + square *rsmt2d.ExtendedDataSquare, +) (file.EdsFile, error) { + tNow := time.Now() + lock := s.stripLock.byDatahashAndHeight(datahash, height) + lock.lock() + defer lock.unlock() + + if datahash.IsEmptyRoot() { + s.addEmptyHeight(height) + return emptyFile, nil + } + + // short circuit if file exists + if has, _ := s.hasByHeight(height); has { + s.metrics.observePutExist(ctx) + return s.getByHeight(height) + } + + filePath := s.basepath + blocksPath + datahash.String() + f, err := s.createFile(filePath, datahash, square) + if err != nil { + s.metrics.observePut(ctx, time.Since(tNow), square.Width(), true) + return nil, fmt.Errorf("creating file: %w", err) + } + + // create hard link with height as name + err = s.createHeightLink(datahash, height) + if err != nil { + s.metrics.observePut(ctx, time.Since(tNow), square.Width(), false) + return nil, fmt.Errorf("linking height: %w", err) + } + s.metrics.observePut(ctx, time.Since(tNow), square.Width(), false) + + // put file in recent cache + f, err = s.cache.First().GetOrLoad(ctx, height, fileLoader(f)) + if err != nil { + log.Warnf("failed to put file in recent cache: %s", err) + } + return f, nil +} + +func (s *Store) createFile(filePath string, datahash share.DataHash, square *rsmt2d.ExtendedDataSquare) (file.EdsFile, error) { + // check if file with the same hash already exists + f, err := s.getByHash(datahash) + if err == nil { + return f, nil + } + + if !errors.Is(err, ErrNotFound) { + return nil, fmt.Errorf("getting by hash: %w", err) + } + + // create Q1Q4 file + f, err = file.CreateQ1Q4File(filePath, datahash, square) + if err != nil { + return nil, fmt.Errorf("creating ODS file: %w", err) + } + return f, nil +} + +func (s *Store) GetByHash(ctx context.Context, datahash share.DataHash) (file.EdsFile, error) { + if datahash.IsEmptyRoot() { + return emptyFile, nil + } + lock := s.stripLock.byDatahash(datahash) + lock.RLock() + defer lock.RUnlock() + + tNow := time.Now() + f, err := s.getByHash(datahash) + s.metrics.observeGet(ctx, time.Since(tNow), err != nil) + return wrappedFile(f), err +} + +func (s *Store) getByHash(datahash share.DataHash) (file.EdsFile, error) { + if datahash.IsEmptyRoot() { + return emptyFile, nil + } + + path := s.basepath + blocksPath + datahash.String() + f, err := file.OpenQ1Q4File(path) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("opening ODS file: %w", err) + } + return f, nil +} + +func (s *Store) LinkHashToHeight(_ context.Context, datahash share.DataHash, height uint64) error { + lock := s.stripLock.byDatahashAndHeight(datahash, height) + lock.lock() + defer lock.unlock() + + if datahash.IsEmptyRoot() { + s.addEmptyHeight(height) + return nil + } + + if has, _ := s.hasByHash(datahash); !has { + return errors.New("cannot link non-existing file") + } + return s.createHeightLink(datahash, height) +} + +func (s *Store) createHeightLink(datahash share.DataHash, height uint64) error { + // short circuit if link exists + if has, _ := s.hasByHeight(height); has { + return nil + } + + filePath := s.basepath + blocksPath + datahash.String() + // create hard link with height as name + linkPath := s.basepath + heightsPath + strconv.Itoa(int(height)) + err := os.Link(filePath, linkPath) + if err != nil { + return fmt.Errorf("creating hard link: %w", err) + } + + return nil +} + +func (s *Store) GetByHeight(ctx context.Context, height uint64) (file.EdsFile, error) { + lock := s.stripLock.byHeight(height) + lock.RLock() + defer lock.RUnlock() + + tNow := time.Now() + f, err := s.getByHeight(height) + s.metrics.observeGet(ctx, time.Since(tNow), err != nil) + return wrappedFile(f), err +} + +func (s *Store) getByHeight(height uint64) (file.EdsFile, error) { + if s.isEmptyHeight(height) { + return emptyFile, nil + } + + f, err := s.cache.Get(height) + if err == nil { + return f, nil + } + + path := s.basepath + heightsPath + fmt.Sprintf("%d", height) + f, err = file.OpenQ1Q4File(path) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("opening ODS file: %w", err) + } + return f, nil +} + +func (s *Store) HasByHash(ctx context.Context, datahash share.DataHash) (bool, error) { + if datahash.IsEmptyRoot() { + return true, nil + } + lock := s.stripLock.byDatahash(datahash) + lock.RLock() + defer lock.RUnlock() + + tNow := time.Now() + exist, err := s.hasByHash(datahash) + s.metrics.observeHas(ctx, time.Since(tNow), err != nil) + return exist, err +} + +func (s *Store) hasByHash(datahash share.DataHash) (bool, error) { + if datahash.IsEmptyRoot() { + return true, nil + } + path := s.basepath + blocksPath + datahash.String() + return pathExists(path) +} + +func (s *Store) HasByHeight(ctx context.Context, height uint64) (bool, error) { + lock := s.stripLock.byHeight(height) + lock.RLock() + defer lock.RUnlock() + + tNow := time.Now() + exist, err := s.hasByHeight(height) + s.metrics.observeHas(ctx, time.Since(tNow), err != nil) + return exist, err +} + +func (s *Store) hasByHeight(height uint64) (bool, error) { + if s.isEmptyHeight(height) { + return true, nil + } + + _, err := s.cache.Get(height) + if err == nil { + return true, nil + } + + path := s.basepath + heightsPath + fmt.Sprintf("%d", height) + return pathExists(path) +} + +func (s *Store) Remove(ctx context.Context, height uint64) error { + lock := s.stripLock.byHeight(height) + lock.Lock() + defer lock.Unlock() + + tNow := time.Now() + err := s.remove(height) + s.metrics.observeRemove(ctx, time.Since(tNow), err != nil) + return err +} + +func (s *Store) remove(height uint64) error { + f, err := s.getByHeight(height) + if err != nil { + // short circuit if file not exists + if errors.Is(err, ErrNotFound) { + return nil + } + return fmt.Errorf("getting by height: %w", err) + } + + // close file to release the reference in the cache + if err = f.Close(); err != nil { + return fmt.Errorf("closing file on removal: %w", err) + } + + if err = s.cache.Remove(height); err != nil { + return fmt.Errorf("removing from cache: %w", err) + } + + // additionally lock by datahash to prevent concurrent access to the same underlying file + // using links from different heights + dlock := s.stripLock.byDatahash(f.DataHash()) + dlock.Lock() + defer dlock.Unlock() + + // remove hard link by height + heightPath := s.basepath + heightsPath + fmt.Sprintf("%d", height) + if err = os.Remove(heightPath); err != nil { + return fmt.Errorf("removing by height: %w", err) + } + + hashStr := f.DataHash().String() + hashPath := s.basepath + blocksPath + hashStr + count, err := linksCount(hashPath) + if err != nil { + return fmt.Errorf("counting links: %w", err) + } + if count == 1 { + err = os.Remove(hashPath) + if err != nil { + return fmt.Errorf("removing by hash: %w", err) + } + } + return nil +} + +func fileLoader(f file.EdsFile) cache.OpenFileFn { + return func(ctx context.Context) (file.EdsFile, error) { + withCache := file.WithProofsCache(f) + return wrappedFile(withCache), nil + } +} + +func wrappedFile(f file.EdsFile) file.EdsFile { + closedOnce := file.WithClosedOnce(f) + sanityChecked := file.WithValidation(closedOnce) + return sanityChecked +} + +func ensureFolder(path string) error { + info, err := os.Stat(path) + if os.IsNotExist(err) { + err = os.Mkdir(path, defaultDirPerm) + if err != nil { + return fmt.Errorf("creating blocks dir: %w", err) + } + return nil + } + if err != nil { + return fmt.Errorf("checking dir: %w", err) + } + if !info.IsDir() { + return errors.New("expected dir, got a file") + } + return nil +} + +func ensureFile(path string) error { + info, err := os.Stat(path) + if os.IsNotExist(err) { + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("creating file: %w", err) + } + return file.Close() + } + if err != nil { + return fmt.Errorf("checking file: %w", err) + } + if info.IsDir() { + return errors.New("expected file, got a dir") + } + return nil +} + +func pathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func linksCount(path string) (int, error) { + info, err := os.Stat(path) + if err != nil { + return 0, fmt.Errorf("checking file: %w", err) + } + + return int(info.Sys().(*syscall.Stat_t).Nlink), nil +} + +func (s *Store) storeEmptyHeights() error { + file, err := os.OpenFile(s.basepath+emptyHeightsFile, os.O_WRONLY, os.ModePerm) + if err != nil { + return fmt.Errorf("opening empty heights file: %w", err) + } + defer utils.CloseAndLog(log, "empty heights file", file) + + s.emptyHeightsLock.RLock() + defer s.emptyHeightsLock.RUnlock() + + encoder := gob.NewEncoder(file) + if err := encoder.Encode(s.emptyHeights); err != nil { + return fmt.Errorf("encoding empty heights: %w", err) + } + + return nil +} + +func loadEmptyHeights(basepath string) (map[uint64]struct{}, error) { + file, err := os.Open(basepath + emptyHeightsFile) + if err != nil { + return nil, fmt.Errorf("opening empty heights file: %w", err) + } + defer utils.CloseAndLog(log, "empty heights file", file) + + emptyHeights := make(map[uint64]struct{}) + err = gob.NewDecoder(file).Decode(&emptyHeights) + if err != nil && !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("decoding empty heights file: %w", err) + } + return emptyHeights, nil +} + +func (s *Store) isEmptyHeight(height uint64) bool { + s.emptyHeightsLock.RLock() + defer s.emptyHeightsLock.RUnlock() + _, ok := s.emptyHeights[height] + return ok +} + +func (s *Store) addEmptyHeight(height uint64) { + s.emptyHeightsLock.Lock() + defer s.emptyHeightsLock.Unlock() + s.emptyHeights[height] = struct{}{} +} diff --git a/share/eds/store_options.go b/share/store/store_options.go similarity index 66% rename from share/eds/store_options.go rename to share/store/store_options.go index c8dcc69136..434badc293 100644 --- a/share/eds/store_options.go +++ b/share/store/store_options.go @@ -1,16 +1,10 @@ -package eds +package store import ( "errors" - "time" ) type Parameters struct { - // GC performs DAG store garbage collection by reclaiming transient files of - // shards that are currently available but inactive, or errored. - // We don't use transient files right now, so GC is turned off by default. - GCInterval time.Duration - // RecentBlocksCacheSize is the size of the cache for recent blocks. RecentBlocksCacheSize int @@ -21,17 +15,12 @@ type Parameters struct { // DefaultParameters returns the default configuration values for the EDS store parameters. func DefaultParameters() *Parameters { return &Parameters{ - GCInterval: 0, RecentBlocksCacheSize: 10, BlockstoreCacheSize: 128, } } func (p *Parameters) Validate() error { - if p.GCInterval < 0 { - return errors.New("eds: GC interval cannot be negative") - } - if p.RecentBlocksCacheSize < 1 { return errors.New("eds: recent blocks cache size must be positive") } diff --git a/share/store/store_test.go b/share/store/store_test.go new file mode 100644 index 0000000000..871e0ee3a0 --- /dev/null +++ b/share/store/store_test.go @@ -0,0 +1,355 @@ +package store + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/libs/rand" + "go.uber.org/atomic" + + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/store/cache" + "github.com/celestiaorg/celestia-node/share/testing/edstest" +) + +//TODO: add benchmarks for store + +func TestEDSStore(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + t.Cleanup(cancel) + + edsStore, err := NewStore(DefaultParameters(), t.TempDir()) + require.NoError(t, err) + + // disable cache + edsStore.cache = cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{}) + height := atomic.NewUint64(100) + + // PutRegistersShard tests if Put registers the shard on the underlying DAGStore + t.Run("Put", func(t *testing.T) { + eds, dah := randomEDS(t) + height := height.Add(1) + + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + + // file should become available by hash + has, err := edsStore.HasByHash(ctx, dah.Hash()) + require.NoError(t, err) + require.True(t, has) + + // file should become available by height + has, err = edsStore.HasByHeight(ctx, height) + require.NoError(t, err) + require.True(t, has) + }) + + t.Run("Cached after Put", func(t *testing.T) { + edsStore, err := NewStore(DefaultParameters(), t.TempDir()) + require.NoError(t, err) + + eds, dah := randomEDS(t) + height := height.Add(1) + + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + + // file should be cached after put + f, err = edsStore.cache.Get(height) + require.NoError(t, err) + require.NoError(t, f.Close()) + + // check that cached file is the same eds + fromFile, err := f.EDS(ctx) + require.NoError(t, err) + require.NoError(t, f.Close()) + require.True(t, eds.Equals(fromFile)) + }) + + t.Run("Second Put should be noop", func(t *testing.T) { + eds, dah := randomEDS(t) + height := height.Add(1) + + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + + f, err = edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + }) + + t.Run("Put eds with same hash for different height", func(t *testing.T) { + eds, dah := randomEDS(t) + h1 := height.Add(1) + + f, err := edsStore.Put(ctx, dah.Hash(), h1, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + + h2 := height.Add(1) + f, err = edsStore.Put(ctx, dah.Hash(), h2, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + + // both heights should be available + has, err := edsStore.HasByHeight(ctx, h1) + require.NoError(t, err) + require.True(t, has) + + has, err = edsStore.HasByHeight(ctx, h2) + require.NoError(t, err) + require.True(t, has) + + // removing one height should not affect the other + err = edsStore.Remove(ctx, h1) + require.NoError(t, err) + + has, err = edsStore.HasByHeight(ctx, h1) + require.NoError(t, err) + require.False(t, has) + + has, err = edsStore.HasByHeight(ctx, h2) + require.NoError(t, err) + require.True(t, has) + }) + + t.Run("GetByHeight", func(t *testing.T) { + eds, dah := randomEDS(t) + height := height.Add(1) + + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + + f, err = edsStore.GetByHeight(ctx, height) + require.NoError(t, err) + + fileEds, err := f.EDS(ctx) + require.NoError(t, err) + require.NoError(t, f.Close()) + + require.True(t, eds.Equals(fileEds)) + }) + + t.Run("GetByDataHash", func(t *testing.T) { + eds, dah := randomEDS(t) + height := height.Add(1) + + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + + f, err = edsStore.GetByHash(ctx, dah.Hash()) + require.NoError(t, err) + + fromFile, err := f.EDS(ctx) + require.NoError(t, err) + require.NoError(t, f.Close()) + + require.True(t, eds.Equals(fromFile)) + }) + + t.Run("Does not exist", func(t *testing.T) { + _, dah := randomEDS(t) + height := height.Add(1) + + has, err := edsStore.HasByHash(ctx, dah.Hash()) + require.NoError(t, err) + require.False(t, has) + + has, err = edsStore.HasByHeight(ctx, height) + require.NoError(t, err) + require.False(t, has) + + _, err = edsStore.GetByHeight(ctx, height) + require.ErrorIs(t, err, ErrNotFound) + + _, err = edsStore.GetByHash(ctx, dah.Hash()) + require.ErrorIs(t, err, ErrNotFound) + }) + + t.Run("Remove", func(t *testing.T) { + // removing file that not exists should be noop + missingHeight := height.Add(1) + err := edsStore.Remove(ctx, missingHeight) + require.NoError(t, err) + + eds, dah := randomEDS(t) + height := height.Add(1) + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + + err = edsStore.Remove(ctx, height) + require.NoError(t, err) + + // file should be removed from cache + _, err = edsStore.cache.Get(height) + require.ErrorIs(t, err, cache.ErrCacheMiss) + + // file should not be accessible by hash + has, err := edsStore.HasByHash(ctx, dah.Hash()) + require.NoError(t, err) + require.False(t, has) + + // file should not be accessible by height + has, err = edsStore.HasByHeight(ctx, height) + require.NoError(t, err) + require.False(t, has) + }) + + t.Run("empty EDS returned by hash", func(t *testing.T) { + eds := share.EmptyExtendedDataSquare() + dah, err := share.NewRoot(eds) + require.NoError(t, err) + + // assert that the empty file exists + has, err := edsStore.HasByHash(ctx, dah.Hash()) + require.NoError(t, err) + require.True(t, has) + + // assert that the empty file is, in fact, empty + f, err := edsStore.GetByHash(ctx, dah.Hash()) + require.NoError(t, err) + require.True(t, f.DataHash().IsEmptyRoot()) + }) + + t.Run("empty EDS returned by height", func(t *testing.T) { + eds := share.EmptyExtendedDataSquare() + dah, err := share.NewRoot(eds) + require.NoError(t, err) + height := height.Add(1) + + // assert that the empty file exists + has, err := edsStore.HasByHeight(ctx, height) + require.NoError(t, err) + require.False(t, has) + + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(t, err) + require.True(t, f.DataHash().IsEmptyRoot()) + require.NoError(t, f.Close()) + + // assert that the empty file can be accessed by height + f, err = edsStore.GetByHeight(ctx, height) + require.NoError(t, err) + require.True(t, f.DataHash().IsEmptyRoot()) + require.NoError(t, f.Close()) + }) + + t.Run("empty EDS are persisted", func(t *testing.T) { + dir := t.TempDir() + edsStore, err := NewStore(DefaultParameters(), dir) + require.NoError(t, err) + + eds := share.EmptyExtendedDataSquare() + dah, err := share.NewRoot(eds) + require.NoError(t, err) + from, to := 10, 20 + + // store empty EDSs + for i := from; i <= to; i++ { + f, err := edsStore.Put(ctx, dah.Hash(), uint64(i), eds) + require.NoError(t, err) + require.NoError(t, f.Close()) + } + + // close and reopen the store to ensure that the empty files are persisted + require.NoError(t, edsStore.Close()) + edsStore, err = NewStore(DefaultParameters(), dir) + require.NoError(t, err) + + // assert that the empty files restored from disk + for i := from; i <= to; i++ { + f, err := edsStore.GetByHeight(ctx, uint64(i)) + require.NoError(t, err) + require.True(t, f.DataHash().IsEmptyRoot()) + require.NoError(t, f.Close()) + } + }) +} + +func BenchmarkStore(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + b.Cleanup(cancel) + + edsStore, err := NewStore(DefaultParameters(), b.TempDir()) + require.NoError(b, err) + + eds := edstest.RandEDS(b, 128) + require.NoError(b, err) + + // BenchmarkStore/bench_put_128-10 27 43968818 ns/op (~43ms) + b.Run("put 128", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + h := share.DataHash(rand.Bytes(5)) + f, _ := edsStore.Put(ctx, h, uint64(i), eds) + _ = f.Close() + } + }) + + // read 128 EDSs does not read full EDS, but only the header + // BenchmarkStore/bench_read_128-10 82766 14678 ns/op (~14ms) + b.Run("open by height, 128", func(b *testing.B) { + edsStore, err := NewStore(DefaultParameters(), b.TempDir()) + require.NoError(b, err) + + // disable cache + edsStore.cache = cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{}) + + dah, err := share.NewRoot(eds) + require.NoError(b, err) + + height := uint64(1984) + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(b, err) + require.NoError(b, f.Close()) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + f, err := edsStore.GetByHeight(ctx, height) + require.NoError(b, err) + require.NoError(b, f.Close()) + } + }) + + // BenchmarkStore/open_by_hash,_128-10 72921 16799 ns/op (~16ms) + b.Run("open by hash, 128", func(b *testing.B) { + edsStore, err := NewStore(DefaultParameters(), b.TempDir()) + require.NoError(b, err) + + // disable cache + edsStore.cache = cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{}) + + dah, err := share.NewRoot(eds) + require.NoError(b, err) + + height := uint64(1984) + f, err := edsStore.Put(ctx, dah.Hash(), height, eds) + require.NoError(b, err) + require.NoError(b, f.Close()) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + f, err := edsStore.GetByHash(ctx, dah.Hash()) + require.NoError(b, err) + require.NoError(b, f.Close()) + } + }) +} + +func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.Root) { + eds := edstest.RandEDS(t, 4) + dah, err := share.NewRoot(eds) + require.NoError(t, err) + + return eds, dah +} diff --git a/share/store/striplock.go b/share/store/striplock.go new file mode 100644 index 0000000000..69cee69f2d --- /dev/null +++ b/share/store/striplock.go @@ -0,0 +1,55 @@ +package store + +import ( + "sync" + + "github.com/celestiaorg/celestia-node/share" +) + +// TODO: move to utils +type striplock struct { + heights []*sync.RWMutex + datahashes []*sync.RWMutex +} + +type multiLock struct { + mu []*sync.RWMutex +} + +func newStripLock(size int) *striplock { + heights := make([]*sync.RWMutex, size) + datahashes := make([]*sync.RWMutex, size) + for i := 0; i < size; i++ { + heights[i] = &sync.RWMutex{} + datahashes[i] = &sync.RWMutex{} + } + return &striplock{heights, datahashes} +} + +func (l *striplock) byHeight(height uint64) *sync.RWMutex { + lkIdx := height % uint64(len(l.heights)) + return l.heights[lkIdx] +} + +func (l *striplock) byDatahash(datahash share.DataHash) *sync.RWMutex { + // Use the last 2 bytes of the datahash as hash to distribute the locks + last := uint16(datahash[len(datahash)-1]) | uint16(datahash[len(datahash)-2])<<8 + lkIdx := last % uint16(len(l.datahashes)) + return l.datahashes[lkIdx] +} + +func (l *striplock) byDatahashAndHeight(datahash share.DataHash, height uint64) *multiLock { + return &multiLock{[]*sync.RWMutex{l.byDatahash(datahash), l.byHeight(height)}} +} + +func (m *multiLock) lock() { + for _, lk := range m.mu { + lk.Lock() + } +} + +func (m *multiLock) unlock() { + for _, lk := range m.mu { + lk.Unlock() + } +} diff --git a/share/eds/edstest/testing.go b/share/testing/edstest/eds.go similarity index 83% rename from share/eds/edstest/testing.go rename to share/testing/edstest/eds.go index bf5e664f90..9ffc93e303 100644 --- a/share/eds/edstest/testing.go +++ b/share/testing/edstest/eds.go @@ -10,7 +10,7 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/sharetest" + "github.com/celestiaorg/celestia-node/share/testing/sharetest" ) func RandByzantineEDS(t *testing.T, size int, options ...nmt.Option) *rsmt2d.ExtendedDataSquare { @@ -34,12 +34,14 @@ func RandEDS(t require.TestingT, size int) *rsmt2d.ExtendedDataSquare { return eds } +// RandEDSWithNamespace generates EDS with given square size. Returned EDS will have +// namespacedAmount of shares with the given namespace. func RandEDSWithNamespace( t require.TestingT, namespace share.Namespace, - size int, + namespacedAmount, size int, ) (*rsmt2d.ExtendedDataSquare, *share.Root) { - shares := sharetest.RandSharesWithNamespace(t, namespace, size*size) + shares := sharetest.RandSharesWithNamespace(t, namespace, namespacedAmount, size*size) eds, err := rsmt2d.ComputeExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) require.NoError(t, err, "failure to recompute the extended data square") dah, err := share.NewRoot(eds) diff --git a/share/sharetest/testing.go b/share/testing/sharetest/share.go similarity index 86% rename from share/sharetest/testing.go rename to share/testing/sharetest/share.go index 3889260393..6564af9b06 100644 --- a/share/sharetest/testing.go +++ b/share/testing/sharetest/share.go @@ -38,17 +38,26 @@ func RandShares(t require.TestingT, total int) []share.Share { } // RandSharesWithNamespace is same the as RandShares, but sets same namespace for all shares. -func RandSharesWithNamespace(t require.TestingT, namespace share.Namespace, total int) []share.Share { +func RandSharesWithNamespace(t require.TestingT, namespace share.Namespace, namespacedAmount, total int) []share.Share { if total&(total-1) != 0 { t.Errorf("total must be power of 2: %d", total) t.FailNow() } + if namespacedAmount > total { + t.Errorf("withNamespace must be less than total: %d", total) + t.FailNow() + } + shares := make([]share.Share, total) rnd := rand.New(rand.NewSource(time.Now().Unix())) //nolint:gosec for i := range shares { shr := make([]byte, share.Size) - copy(share.GetNamespace(shr), namespace) + if i < namespacedAmount { + copy(share.GetNamespace(shr), namespace) + } else { + copy(share.GetNamespace(shr), RandV0Namespace()) + } _, err := rnd.Read(share.GetData(shr)) require.NoError(t, err) shares[i] = shr diff --git a/share/utils.go b/share/utils.go new file mode 100644 index 0000000000..db12b039fd --- /dev/null +++ b/share/utils.go @@ -0,0 +1,18 @@ +package share + +// TODO(@walldiss): refactor this into proper package once we have a better idea of what it should look like +func RowRangeForNamespace(root *Root, namespace Namespace) (from, to int) { + from = -1 + for i, row := range root.RowRoots { + if !namespace.IsOutsideRange(row, row) { + if from == -1 { + from = i + } + to = i + 1 + } + } + if to == 0 { + return 0, 0 + } + return from, to +}