diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 73572e4cef..e914c967d0 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -4,9 +4,8 @@ on: push: branches: - main - # Trigger on version tags - tags: - - "v*" + release: + types: [published] pull_request: workflow_dispatch: inputs: @@ -29,7 +28,7 @@ jobs: runs-on: ubuntu-latest env: # upgrade go version throughout pipeline here - GO_VERSION: "1.21" + GO_VERSION: "1.22" outputs: go-version: ${{ steps.set-vars.outputs.go-version }} branch: ${{ steps.trim_ref.outputs.branch }} @@ -99,7 +98,7 @@ jobs: # version_bump section, it would skip and not run, which would result # in goreleaser not running either. if: ${{ github.event_name == 'workflow_dispatch' }} - uses: mathieudutour/github-tag-action@v6.1 + uses: mathieudutour/github-tag-action@v6.2 with: github_token: ${{ secrets.GITHUB_TOKEN }} default_bump: ${{ inputs.version }} @@ -111,7 +110,7 @@ jobs: runs-on: ubuntu-latest if: | github.event_name == 'workflow_dispatch' || - (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) + github.event_name == 'release' permissions: "write-all" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 0098c3d18a..0bcd33b2f6 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -44,7 +44,7 @@ jobs: uses: golangci/golangci-lint-action@v4.0.0 with: args: --timeout 10m - version: v1.55 + version: v1.56 skip-pkg-cache: true skip-build-cache: true @@ -98,7 +98,7 @@ jobs: retention-days: 5 - name: upload coverage - uses: codecov/codecov-action@v4.0.2 + uses: codecov/codecov-action@v4.1.0 with: env_vars: OS token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.golangci.yml b/.golangci.yml index a0f2754a9b..bcc34b20f0 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -48,6 +48,7 @@ issues: - path: _test\.go linters: - gosec + - revive - linters: - lll source: "https://" diff --git a/Dockerfile b/Dockerfile index 01fccafe2e..5e29d82c43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM docker.io/golang:1.21-alpine3.18 as builder +FROM --platform=$BUILDPLATFORM docker.io/golang:1.22-alpine3.18 as builder ARG TARGETPLATFORM ARG BUILDPLATFORM @@ -40,16 +40,16 @@ ENV P2P_NETWORK mocha # hadolint ignore=DL3018 RUN uname -a &&\ apk update && apk add --no-cache \ - bash \ - curl \ - jq \ + bash \ + curl \ + jq \ # Creates a user with $UID and $GID=$UID && adduser ${USER_NAME} \ - -D \ - -g ${USER_NAME} \ - -h ${CELESTIA_HOME} \ - -s /sbin/nologin \ - -u ${UID} + -D \ + -g ${USER_NAME} \ + -h ${CELESTIA_HOME} \ + -s /sbin/nologin \ + -u ${UID} # Copy in the binary COPY --from=builder /src/build/celestia /bin/celestia @@ -59,6 +59,8 @@ COPY --chown=${USER_NAME}:${USER_NAME} docker/entrypoint.sh /opt/entrypoint.sh USER ${USER_NAME} +WORKDIR ${CELESTIA_HOME} + EXPOSE 2121 ENTRYPOINT [ "/bin/bash", "/opt/entrypoint.sh" ] diff --git a/Makefile b/Makefile index a43c917345..ddc0b93a6b 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ SHELL=/usr/bin/env bash PROJECTNAME=$(shell basename "$(PWD)") DIR_FULLPATH=$(shell pwd) versioningPath := "github.com/celestiaorg/celestia-node/nodebuilder/node" +OS := $(shell uname -s) LDFLAGS=-ldflags="-X '$(versioningPath).buildTime=$(shell date)' -X '$(versioningPath).lastCommit=$(shell git rev-parse HEAD)' -X '$(versioningPath).semanticVersion=$(shell git describe --tags --dirty=-dev 2>/dev/null || git rev-parse --abbrev-ref HEAD)'" TAGS=integration SHORT= @@ -63,11 +64,19 @@ deps: @go mod download .PHONY: deps -## install: Install all build binaries into the $PREFIX (/usr/local/ by default) directory. +## install: Install the celestia-node binary. install: +ifeq ($(OS),Darwin) + @$(MAKE) go-install +else + @$(MAKE) install-global +endif +.PHONY: install + +install-global: @echo "--> Installing Celestia" @install -v ./build/* -t ${PREFIX}/bin/ -.PHONY: install +.PHONY: install-global ## go-install: Build and install the celestia-node binary into the GOBIN directory. go-install: diff --git a/README.md b/README.md index 3a552495ad..423ec17615 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Continue reading [here](https://blog.celestia.org/celestia-mvp-release-data-avai | Requirement | Notes | | ----------- |----------------| -| Go version | 1.21 or higher | +| Go version | 1.22 or higher | ## System Requirements diff --git a/api/docgen/openrpc.go b/api/docgen/openrpc.go index 737b491a89..4ec98d73b0 100644 --- a/api/docgen/openrpc.go +++ b/api/docgen/openrpc.go @@ -1,5 +1,7 @@ // Package docgen generates an OpenRPC spec for the Celestia Node. It has been inspired by and // adapted from Filecoin's Lotus API implementation. + +//nolint:revive package docgen import ( @@ -95,7 +97,7 @@ func NewOpenRPCDocument(comments Comments, permissions Comments) *go_openrpc_ref d.WithMeta(&go_openrpc_reflect.MetaT{ GetServersFn: func() func(listeners []net.Listener) (*meta_schema.Servers, error) { - return func(listeners []net.Listener) (*meta_schema.Servers, error) { + return func(_ []net.Listener) (*meta_schema.Servers, error) { return nil, nil } }, diff --git a/blob/blob.go b/blob/blob.go index 9843441dd2..c81d83070e 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -9,13 +9,14 @@ import ( tmproto "github.com/tendermint/tendermint/proto/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/nmt" "github.com/celestiaorg/celestia-node/share" ) +var errEmptyShares = errors.New("empty shares") + // Commitment is a Merkle Root of the subtree built from shares of the Blob. // It is computed by splitting the blob into shares and building the Merkle subtree to be included // after Submit. @@ -31,37 +32,13 @@ func (com Commitment) Equal(c Commitment) bool { } // Proof is a collection of nmt.Proofs that verifies the inclusion of the data. +// Proof proves the WHOLE namespaced data for the particular row. +// TODO (@vgonkivs): rework `Proof` in order to prove a particular blob. +// https://github.com/celestiaorg/celestia-node/issues/2303 type Proof []*nmt.Proof func (p Proof) Len() int { return len(p) } -func (p Proof) MarshalJSON() ([]byte, error) { - proofs := make([]string, 0, len(p)) - for _, proof := range p { - proofBytes, err := proof.MarshalJSON() - if err != nil { - return nil, err - } - proofs = append(proofs, string(proofBytes)) - } - return json.Marshal(proofs) -} - -func (p *Proof) UnmarshalJSON(b []byte) error { - var proofs []string - if err := json.Unmarshal(b, &proofs); err != nil { - return err - } - for _, proof := range proofs { - var nmtProof nmt.Proof - if err := nmtProof.UnmarshalJSON([]byte(proof)); err != nil { - return err - } - *p = append(*p, &nmtProof) - } - return nil -} - // equal is a temporary method that compares two proofs. // should be removed in BlobService V1. func (p Proof) equal(input Proof) error { @@ -99,6 +76,10 @@ type Blob struct { // the celestia-node's namespace type // this is to avoid converting to and from app's type namespace share.Namespace + + // index represents the index of the blob's first share in the EDS. + // Only retrieved, on-chain blobs will have the index set. Default is -1. + index int } // NewBlobV0 constructs a new blob from the provided Namespace and data. @@ -127,7 +108,7 @@ func NewBlob(shareVersion uint8, namespace share.Namespace, data []byte) (*Blob, if err != nil { return nil, err } - return &Blob{Blob: blob, Commitment: com, namespace: namespace}, nil + return &Blob{Blob: blob, Commitment: com, namespace: namespace, index: -1}, nil } // Namespace returns blob's namespace. @@ -135,11 +116,22 @@ func (b *Blob) Namespace() share.Namespace { return b.namespace } +// Index returns the blob's first share index in the EDS. +// Only retrieved, on-chain blobs will have the index set. Default is -1. +func (b *Blob) Index() int { + return b.index +} + +func (b *Blob) compareCommitments(com Commitment) bool { + return bytes.Equal(b.Commitment, com) +} + type jsonBlob struct { Namespace share.Namespace `json:"namespace"` Data []byte `json:"data"` ShareVersion uint32 `json:"share_version"` Commitment Commitment `json:"commitment"` + Index int `json:"index"` } func (b *Blob) MarshalJSON() ([]byte, error) { @@ -148,6 +140,7 @@ func (b *Blob) MarshalJSON() ([]byte, error) { Data: b.Data, ShareVersion: b.ShareVersion, Commitment: b.Commitment, + Index: b.index, } return json.Marshal(blob) } @@ -165,39 +158,6 @@ func (b *Blob) UnmarshalJSON(data []byte) error { b.Blob.ShareVersion = blob.ShareVersion b.Commitment = blob.Commitment b.namespace = blob.Namespace + b.index = blob.Index return nil } - -// buildBlobsIfExist takes shares and tries building the Blobs from them. -// It will build blobs either until appShares will be empty or the first incomplete blob will -// appear, so in this specific case it will return all built blobs + remaining shares. -func buildBlobsIfExist(appShares []shares.Share) ([]*Blob, []shares.Share, error) { - if len(appShares) == 0 { - return nil, nil, errors.New("empty shares received") - } - blobs := make([]*Blob, 0, len(appShares)) - for { - length, err := appShares[0].SequenceLen() - if err != nil { - return nil, nil, err - } - - amount := shares.SparseSharesNeeded(length) - if amount > len(appShares) { - return blobs, appShares, nil - } - - b, err := parseShares(appShares[:amount]) - if err != nil { - return nil, nil, err - } - - // only 1 blob will be created bc we passed the exact amount of shares - blobs = append(blobs, b[0]) - - if amount == len(appShares) { - return blobs, nil, nil - } - appShares = appShares[amount:] - } -} diff --git a/blob/blob_test.go b/blob/blob_test.go index 85486ad125..020bfb191b 100644 --- a/blob/blob_test.go +++ b/blob/blob_test.go @@ -1,7 +1,7 @@ package blob import ( - "reflect" + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -14,7 +14,7 @@ import ( ) func TestBlob(t *testing.T) { - appBlobs, err := blobtest.GenerateV0Blobs([]int{1}, false) + appBlobs, err := blobtest.GenerateV0Blobs([]int{16}, false) require.NoError(t, err) blob, err := convertBlobs(appBlobs...) require.NoError(t, err) @@ -53,10 +53,12 @@ func TestBlob(t *testing.T) { expectedRes: func(t *testing.T) { sh, err := BlobsToShares(blob...) require.NoError(t, err) - b, err := SharesToBlobs(sh) + shares, err := toAppShares(sh...) require.NoError(t, err) - assert.Equal(t, len(b), 1) - assert.Equal(t, blob[0].Commitment, b[0].Commitment) + p := &parser{length: len(shares), shares: shares} + b, err := p.parse() + require.NoError(t, err) + assert.Equal(t, blob[0].Commitment, b.Commitment) }, }, { @@ -67,7 +69,8 @@ func TestBlob(t *testing.T) { newBlob := &Blob{} require.NoError(t, newBlob.UnmarshalJSON(data)) - require.True(t, reflect.DeepEqual(blob[0], newBlob)) + require.True(t, bytes.Equal(blob[0].Blob.Data, newBlob.Data)) + require.True(t, bytes.Equal(blob[0].Commitment, newBlob.Commitment)) }, }, } diff --git a/blob/helper.go b/blob/helper.go index 72a56c7889..d3b418e8ba 100644 --- a/blob/helper.go +++ b/blob/helper.go @@ -11,55 +11,6 @@ import ( "github.com/celestiaorg/celestia-node/share" ) -// SharesToBlobs takes raw shares and converts them to the blobs. -func SharesToBlobs(rawShares []share.Share) ([]*Blob, error) { - if len(rawShares) == 0 { - return nil, ErrBlobNotFound - } - - appShares, err := toAppShares(rawShares...) - if err != nil { - return nil, err - } - return parseShares(appShares) -} - -// parseShares takes shares and converts them to the []*Blob. -func parseShares(appShrs []shares.Share) ([]*Blob, error) { - shareSequences, err := shares.ParseShares(appShrs, true) - if err != nil { - return nil, err - } - - // ensure that sequence length is not 0 - if len(shareSequences) == 0 { - return nil, ErrBlobNotFound - } - - blobs := make([]*Blob, len(shareSequences)) - for i, sequence := range shareSequences { - data, err := sequence.RawData() - if err != nil { - return nil, err - } - if len(data) == 0 { - continue - } - - shareVersion, err := sequence.Shares[0].Version() - if err != nil { - return nil, err - } - - blob, err := NewBlob(shareVersion, sequence.Namespace.Bytes(), data) - if err != nil { - return nil, err - } - blobs[i] = blob - } - return blobs, nil -} - // BlobsToShares accepts blobs and convert them to the Shares. func BlobsToShares(blobs ...*Blob) ([]share.Share, error) { b := make([]types.Blob, len(blobs)) @@ -75,7 +26,7 @@ func BlobsToShares(blobs ...*Blob) ([]share.Share, error) { sort.Slice(b, func(i, j int) bool { val := bytes.Compare(b[i].NamespaceID, b[j].NamespaceID) - return val <= 0 + return val < 0 }) rawShares, err := shares.SplitBlobs(b...) @@ -84,3 +35,22 @@ func BlobsToShares(blobs ...*Blob) ([]share.Share, error) { } return shares.ToBytes(rawShares), nil } + +// toAppShares converts node's raw shares to the app shares, skipping padding +func toAppShares(shrs ...share.Share) ([]shares.Share, error) { + appShrs := make([]shares.Share, 0, len(shrs)) + for _, shr := range shrs { + bShare, err := shares.NewShare(shr) + if err != nil { + return nil, err + } + appShrs = append(appShrs, *bShare) + } + return appShrs, nil +} + +func calculateIndex(rowLength, blobIndex int) (row, col int) { + row = blobIndex / rowLength + col = blobIndex - (row * rowLength) + return +} diff --git a/blob/parser.go b/blob/parser.go new file mode 100644 index 0000000000..8999afcb22 --- /dev/null +++ b/blob/parser.go @@ -0,0 +1,151 @@ +package blob + +import ( + "errors" + "fmt" + + "github.com/celestiaorg/celestia-app/pkg/shares" +) + +// parser is a helper struct that allows collecting shares and transforming them into the blob. +// it contains all necessary information that is needed to build the blob: +// * position of the blob inside the EDS; +// * blob's length; +// * shares needed to build the blob; +// * extra condition to verify the final blob. +type parser struct { + index int + length int + shares []shares.Share + verifyFn func(blob *Blob) bool +} + +// NOTE: passing shares here needed to detect padding shares(as we do not need this check in addShares) +func (p *parser) set(index int, shrs []shares.Share) ([]shares.Share, error) { + if len(shrs) == 0 { + return nil, errEmptyShares + } + + shrs, err := p.skipPadding(shrs) + if err != nil { + return nil, err + } + + if len(shrs) == 0 { + return nil, errEmptyShares + } + + // `+=` as index could be updated in `skipPadding` + p.index += index + length, err := shrs[0].SequenceLen() + if err != nil { + return nil, err + } + + p.length = shares.SparseSharesNeeded(length) + return shrs, nil +} + +// addShares sets shares until the blob is completed and returns extra shares back. +// we do not need here extra condition to check padding shares as we do not expect it here. +// it is possible only between two blobs. +func (p *parser) addShares(shares []shares.Share) (shrs []shares.Share, isComplete bool) { + index := -1 + for i, sh := range shares { + p.shares = append(p.shares, sh) + if len(p.shares) == p.length { + index = i + isComplete = true + break + } + } + + if index == -1 { + return + } + + if index+1 >= len(shares) { + return shrs, true + } + return shares[index+1:], true +} + +// parse parses shares and creates the Blob. +func (p *parser) parse() (*Blob, error) { + if p.length != len(p.shares) { + return nil, fmt.Errorf("invalid shares amount. want:%d, have:%d", p.length, len(p.shares)) + } + + sequence, err := shares.ParseShares(p.shares, true) + if err != nil { + return nil, err + } + + // ensure that sequence length is not 0 + if len(sequence) == 0 { + return nil, ErrBlobNotFound + } + if len(sequence) > 1 { + return nil, errors.New("unexpected amount of sequences") + } + + data, err := sequence[0].RawData() + if err != nil { + return nil, err + } + if len(data) == 0 { + return nil, ErrBlobNotFound + } + + shareVersion, err := sequence[0].Shares[0].Version() + if err != nil { + return nil, err + } + + blob, err := NewBlob(shareVersion, sequence[0].Namespace.Bytes(), data) + if err != nil { + return nil, err + } + blob.index = p.index + return blob, nil +} + +// skipPadding skips first share in the range if this share is the Padding share. +func (p *parser) skipPadding(shares []shares.Share) ([]shares.Share, error) { + if len(shares) == 0 { + return nil, errEmptyShares + } + + isPadding, err := shares[0].IsPadding() + if err != nil { + return nil, err + } + + if !isPadding { + return shares, nil + } + + // update blob index if we are going to skip one share + p.index++ + if len(shares) > 1 { + return shares[1:], nil + } + return nil, nil +} + +func (p *parser) verify(blob *Blob) bool { + if p.verifyFn == nil { + return false + } + return p.verifyFn(blob) +} + +func (p *parser) isEmpty() bool { + return p.index == 0 && p.length == 0 && len(p.shares) == 0 +} + +func (p *parser) reset() { + p.index = 0 + p.length = 0 + p.shares = nil +} diff --git a/blob/service.go b/blob/service.go index fc1d630e62..7445c05054 100644 --- a/blob/service.go +++ b/blob/service.go @@ -111,12 +111,27 @@ func (s *Service) Submit(ctx context.Context, blobs []*Blob, gasPrice GasPrice) } // Get retrieves all the blobs for given namespaces at the given height by commitment. -func (s *Service) Get(ctx context.Context, height uint64, ns share.Namespace, commitment Commitment) (*Blob, error) { - blob, _, err := s.getByCommitment(ctx, height, ns, commitment) - if err != nil { - return nil, err - } - return blob, nil +func (s *Service) Get( + ctx context.Context, + height uint64, + namespace share.Namespace, + commitment Commitment, +) (blob *Blob, err error) { + ctx, span := tracer.Start(ctx, "get") + defer func() { + utils.SetStatusAndEnd(span, err) + }() + span.SetAttributes( + attribute.Int64("height", int64(height)), + attribute.String("namespace", namespace.String()), + ) + + sharesParser := &parser{verifyFn: func(blob *Blob) bool { + return blob.compareCommitments(commitment) + }} + + blob, _, err = s.retrieve(ctx, height, namespace, sharesParser) + return } // GetProof retrieves all blobs in the given namespaces at the given height by commitment @@ -126,11 +141,21 @@ func (s *Service) GetProof( height uint64, namespace share.Namespace, commitment Commitment, -) (*Proof, error) { - _, proof, err := s.getByCommitment(ctx, height, namespace, commitment) - if err != nil { - return nil, err - } +) (proof *Proof, err error) { + ctx, span := tracer.Start(ctx, "get-proof") + defer func() { + utils.SetStatusAndEnd(span, err) + }() + span.SetAttributes( + attribute.Int64("height", int64(height)), + attribute.String("namespace", namespace.String()), + ) + + sharesParser := &parser{verifyFn: func(blob *Blob) bool { + return blob.compareCommitments(commitment) + }} + + _, proof, err = s.retrieve(ctx, height, namespace, sharesParser) return proof, nil } @@ -147,8 +172,8 @@ func (s *Service) GetAll(ctx context.Context, height uint64, namespaces []share. resultErr = make([]error, len(namespaces)) ) - for _, ns := range namespaces { - log.Debugw("performing GetAll request", "namespace", ns.String(), "height", height) + for _, namespace := range namespaces { + log.Debugw("performing GetAll request", "namespace", namespace.String(), "height", height) } wg := sync.WaitGroup{} @@ -190,12 +215,17 @@ func (s *Service) Included( height uint64, namespace share.Namespace, proof *Proof, - com Commitment, + commitment Commitment, ) (_ bool, err error) { ctx, span := tracer.Start(ctx, "included") defer func() { utils.SetStatusAndEnd(span, err) }() + span.SetAttributes( + attribute.Int64("height", int64(height)), + attribute.String("namespace", namespace.String()), + ) + // In the current implementation, LNs will have to download all shares to recompute the commitment. // To achieve 1. we need to modify Proof structure and to store all subtree roots, that were // involved in commitment creation and then call `merkle.HashFromByteSlices`(tendermint package). @@ -205,7 +235,10 @@ func (s *Service) Included( // but we have to guarantee that all our stored subtree roots will be on the same height(e.g. one // level above shares). // TODO(@vgonkivs): rework the implementation to perform all verification without network requests. - _, resProof, err := s.getByCommitment(ctx, height, namespace, com) + sharesParser := &parser{verifyFn: func(blob *Blob) bool { + return blob.compareCommitments(commitment) + }} + _, resProof, err := s.retrieve(ctx, height, namespace, sharesParser) switch err { case nil: case ErrBlobNotFound: @@ -216,27 +249,19 @@ func (s *Service) Included( return true, resProof.equal(*proof) } -// getByCommitment retrieves the DAH row by row, fetching shares and constructing blobs in order to -// compare Commitments. Retrieving is stopped once the requested blob/proof is found. -func (s *Service) getByCommitment( +// retrieve retrieves blobs and their proofs by requesting the whole namespace and +// comparing Commitments with each blob. +// Retrieving is stopped once the requested blob/proof is found. +func (s *Service) retrieve( ctx context.Context, height uint64, namespace share.Namespace, - commitment Commitment, + sharesParser *parser, ) (_ *Blob, _ *Proof, err error) { log.Infow("requesting blob", "height", height, "namespace", namespace.String()) - ctx, span := tracer.Start(ctx, "get-by-commitment") - defer func() { - utils.SetStatusAndEnd(span, err) - }() - span.SetAttributes( - attribute.Int64("height", int64(height)), - attribute.String("commitment", string(commitment)), - ) - getCtx, headerGetterSpan := tracer.Start(ctx, "header-getter") header, err := s.headerGetter(getCtx, height) @@ -249,6 +274,14 @@ func (s *Service) getByCommitment( headerGetterSpan.AddEvent("received eds", trace.WithAttributes( attribute.Int64("eds-size", int64(len(header.DAH.RowRoots))))) + rowIndex := -1 + for i, row := range header.DAH.RowRoots { + if !namespace.IsOutsideRange(row, row) { + rowIndex = i + break + } + } + getCtx, getSharesSpan := tracer.Start(ctx, "get-shares-by-namespace") namespacedShares, err := s.shareGetter.GetSharesByNamespace(getCtx, header, namespace) @@ -265,10 +298,8 @@ func (s *Service) getByCommitment( attribute.Int64("eds-size", int64(len(header.DAH.RowRoots))))) var ( - rawShares = make([]shares.Share, 0) + appShares = make([]shares.Share, 0) proofs = make(Proof, 0) - // spansMultipleRows specifies whether blob is expanded into multiple rows - spansMultipleRows bool ) for _, row := range namespacedShares { @@ -279,47 +310,81 @@ func (s *Service) getByCommitment( return nil, nil, ErrBlobNotFound } - appShares, err := toAppShares(row.Shares...) + appShares, err = toAppShares(row.Shares...) if err != nil { return nil, nil, err } - rawShares = append(rawShares, appShares...) + proofs = append(proofs, row.Proof) + index := row.Proof.Start() + + for { + var ( + isComplete bool + shrs []shares.Share + wasEmpty = sharesParser.isEmpty() + ) + + if wasEmpty { + // create a parser if it is empty + shrs, err = sharesParser.set(rowIndex*len(header.DAH.RowRoots)+index, appShares) + if err != nil { + if errors.Is(err, errEmptyShares) { + appShares = nil + break + } + return nil, nil, err + } + + if len(appShares) != len(shrs) { + // update index and shares if a padding share was detected. + index += len(appShares) - len(shrs) + appShares = shrs + } + } - var blobs []*Blob - blobs, rawShares, err = buildBlobsIfExist(rawShares) - if err != nil { - return nil, nil, err - } - for _, b := range blobs { - if b.Commitment.Equal(commitment) { - span.AddEvent("blob reconstructed") - return b, &proofs, nil + shrs, isComplete = sharesParser.addShares(appShares) + if !isComplete { + appShares = nil + break } - // Falling under this flag means that the data from the last row - // was insufficient to create a complete blob. As a result, - // the first blob received spans two rows and includes proofs - // for both of these rows. All other blobs in the result will relate - // to the current row and have a single proof. - if spansMultipleRows { - spansMultipleRows = false - // leave proof only for the current row + + blob, err := sharesParser.parse() + if err != nil { + return nil, nil, err + } + + if sharesParser.verify(blob) { + return blob, &proofs, nil + } + + index += len(appShares) - len(shrs) + appShares = shrs + sharesParser.reset() + + if !wasEmpty { + // remove proofs for prev rows if verified blob spans multiple rows proofs = proofs[len(proofs)-1:] } } - if len(rawShares) > 0 { - spansMultipleRows = true - continue + rowIndex++ + if sharesParser.isEmpty() { + proofs = nil } - proofs = nil } err = ErrBlobNotFound - if len(rawShares) > 0 { - err = fmt.Errorf("incomplete blob with the "+ - "namespace: %s detected at %d: %w", namespace.String(), height, err) - log.Error(err) + for _, sh := range appShares { + ok, err := sh.IsPadding() + if err != nil { + return nil, nil, err + } + if !ok { + err = fmt.Errorf("incomplete blob with the "+ + "namespace: %s detected at %d: %w", namespace.String(), height, err) + log.Error(err) + } } return nil, nil, err } @@ -332,34 +397,24 @@ func (s *Service) getBlobs( header *header.ExtendedHeader, ) (_ []*Blob, err error) { ctx, span := tracer.Start(ctx, "get-blobs") + span.SetAttributes( + attribute.Int64("height", int64(header.Height())), + attribute.String("namespace", namespace.String()), + ) defer func() { utils.SetStatusAndEnd(span, err) }() - namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header, namespace) - if err != nil { - return nil, err - } - return SharesToBlobs(namespacedShares.Flatten()) -} -// toAppShares converts node's raw shares to the app shares, skipping padding -func toAppShares(shrs ...share.Share) ([]shares.Share, error) { - appShrs := make([]shares.Share, 0, len(shrs)) - for _, shr := range shrs { - bShare, err := shares.NewShare(shr) - if err != nil { - return nil, err - } - - ok, err := bShare.IsPadding() - if err != nil { - return nil, err - } - if ok { - continue - } + blobs := make([]*Blob, 0) + verifyFn := func(blob *Blob) bool { + blobs = append(blobs, blob) + return false + } + sharesParser := &parser{verifyFn: verifyFn} - appShrs = append(appShrs, *bShare) + _, _, err = s.retrieve(ctx, header.Height(), namespace, sharesParser) + if len(blobs) == 0 { + return nil, ErrBlobNotFound } - return appShrs, nil + return blobs, nil } diff --git a/blob/service_test.go b/blob/service_test.go index 3e22f887af..4bda8d993b 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -5,6 +5,8 @@ import ( "context" "crypto/sha256" "encoding/json" + "fmt" + "sort" "testing" "time" @@ -72,8 +74,7 @@ func TestBlobService_Get(t *testing.T) { { name: "get all with the same namespace", doFn: func() (interface{}, error) { - b, err := service.GetAll(ctx, 1, []share.Namespace{blobs1[0].Namespace()}) - return b, err + return service.GetAll(ctx, 1, []share.Namespace{blobs1[0].Namespace()}) }, expectedResult: func(res interface{}, err error) { require.NoError(t, err) @@ -85,7 +86,47 @@ func TestBlobService_Get(t *testing.T) { assert.Len(t, blobs, 2) for i := range blobs1 { - bytes.Equal(blobs1[i].Commitment, blobs[i].Commitment) + require.Equal(t, blobs1[i].Commitment, blobs[i].Commitment) + } + }, + }, + { + name: "verify indexes", + doFn: func() (interface{}, error) { + b0, err := service.Get(ctx, 1, blobs0[0].Namespace(), blobs0[0].Commitment) + require.NoError(t, err) + b1, err := service.Get(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment) + require.NoError(t, err) + b23, err := service.GetAll(ctx, 1, []share.Namespace{blobs1[0].Namespace()}) + require.NoError(t, err) + return []*Blob{b0, b1, b23[0], b23[1]}, nil + }, + expectedResult: func(res interface{}, err error) { + require.NoError(t, err) + blobs, ok := res.([]*Blob) + assert.True(t, ok) + assert.NotEmpty(t, blobs) + assert.Len(t, blobs, 4) + + sort.Slice(blobs, func(i, j int) bool { + val := bytes.Compare(blobs[i].NamespaceId, blobs[j].NamespaceId) + return val < 0 + }) + + h, err := service.headerGetter(ctx, 1) + require.NoError(t, err) + + resultShares, err := BlobsToShares(blobs...) + require.NoError(t, err) + shareOffset := 0 + for i := range blobs { + row, col := calculateIndex(len(h.DAH.RowRoots), blobs[i].index) + sh, err := service.shareGetter.GetShare(ctx, h, row, col) + require.NoError(t, err) + require.True(t, bytes.Equal(sh, resultShares[shareOffset]), + fmt.Sprintf("issue on %d attempt. ROW:%d, COL: %d, blobIndex:%d", i, row, col, blobs[i].index), + ) + shareOffset += shares.SparseSharesNeeded(uint32(len(blobs[i].Data))) } }, }, @@ -300,7 +341,7 @@ func TestBlobService_Get(t *testing.T) { // But to satisfy the rule of eds creating, padding namespace share is placed between // blobs. Test ensures that blob service will skip padding share and return the correct blob. func TestService_GetSingleBlobWithoutPadding(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) t.Cleanup(cancel) appBlob, err := blobtest.GenerateV0Blobs([]int{9, 5}, true) @@ -342,6 +383,14 @@ func TestService_GetSingleBlobWithoutPadding(t *testing.T) { newBlob, err := service.Get(ctx, 1, blobs[1].Namespace(), blobs[1].Commitment) require.NoError(t, err) assert.Equal(t, newBlob.Commitment, blobs[1].Commitment) + + resultShares, err := BlobsToShares(newBlob) + require.NoError(t, err) + row, col := calculateIndex(len(h.DAH.RowRoots), newBlob.index) + sh, err := service.shareGetter.GetShare(ctx, h, row, col) + require.NoError(t, err) + + assert.Equal(t, sh, resultShares[0]) } func TestService_Get(t *testing.T) { @@ -356,10 +405,25 @@ func TestService_Get(t *testing.T) { require.NoError(t, err) service := createService(ctx, t, blobs) - for _, blob := range blobs { + + h, err := service.headerGetter(ctx, 1) + require.NoError(t, err) + + resultShares, err := BlobsToShares(blobs...) + require.NoError(t, err) + shareOffset := 0 + + for i, blob := range blobs { b, err := service.Get(ctx, 1, blob.Namespace(), blob.Commitment) require.NoError(t, err) assert.Equal(t, b.Commitment, blob.Commitment) + + row, col := calculateIndex(len(h.DAH.RowRoots), b.index) + sh, err := service.shareGetter.GetShare(ctx, h, row, col) + require.NoError(t, err) + + assert.Equal(t, sh, resultShares[shareOffset], fmt.Sprintf("issue on %d attempt", i)) + shareOffset += shares.SparseSharesNeeded(uint32(len(blob.Data))) } } @@ -410,11 +474,27 @@ func TestService_GetAllWithoutPadding(t *testing.T) { service := NewService(nil, getters.NewIPLDGetter(bs), fn) - _, err = service.GetAll(ctx, 1, []share.Namespace{blobs[0].Namespace(), blobs[1].Namespace()}) + blobs, err = service.GetAll(ctx, 1, []share.Namespace{blobs[0].Namespace(), blobs[1].Namespace()}) + require.NoError(t, err) + + resultShares, err := BlobsToShares(blobs...) require.NoError(t, err) + sort.Slice(blobs, func(i, j int) bool { + val := bytes.Compare(blobs[i].NamespaceId, blobs[j].NamespaceId) + return val < 0 + }) + shareOffset := 0 + for _, blob := range blobs { + row, col := calculateIndex(len(h.DAH.RowRoots), blob.index) + sh, err := service.shareGetter.GetShare(ctx, h, row, col) + require.NoError(t, err) + + assert.Equal(t, sh, resultShares[shareOffset]) + shareOffset += shares.SparseSharesNeeded(uint32(len(blob.Data))) + } } -// BenchmarkGetByCommitment-12 3139 380827 ns/op 701647 B/op 4990 allocs/op +// BenchmarkGetByCommitment-12 1869 571663 ns/op 1085371 B/op 6414 allocs/op func BenchmarkGetByCommitment(b *testing.B) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) b.Cleanup(cancel) @@ -425,11 +505,17 @@ func BenchmarkGetByCommitment(b *testing.B) { require.NoError(b, err) service := createService(ctx, b, blobs) + indexer := &parser{} b.ResetTimer() for i := 0; i < b.N; i++ { b.ReportAllocs() - _, _, err = service.getByCommitment( - ctx, 1, blobs[1].Namespace(), blobs[1].Commitment, + indexer.reset() + indexer.verifyFn = func(blob *Blob) bool { + return blob.compareCommitments(blobs[1].Commitment) + } + + _, _, err = service.retrieve( + ctx, 1, blobs[1].Namespace(), indexer, ) require.NoError(b, err) } diff --git a/cmd/cel-shed/eds_store_stress.go b/cmd/cel-shed/eds_store_stress.go index 62ea5cb772..9036a81e30 100644 --- a/cmd/cel-shed/eds_store_stress.go +++ b/cmd/cel-shed/eds_store_stress.go @@ -10,9 +10,9 @@ import ( "os" "time" + "github.com/grafana/pyroscope-go" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" - "github.com/pyroscope-io/client/pyroscope" "github.com/spf13/cobra" "github.com/celestiaorg/celestia-node/libs/edssser" @@ -69,7 +69,7 @@ var edsStoreStress = &cobra.Command{ Use: "stress", Short: `Runs eds.Store stress test over default node.Store Datastore backend (e.g. Badger).`, SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) (err error) { + RunE: func(cmd *cobra.Command, _ []string) (err error) { // expose expvar vars over http go http.ListenAndServe(":9999", http.DefaultServeMux) //nolint:errcheck,gosec diff --git a/cmd/cel-shed/p2p.go b/cmd/cel-shed/p2p.go index a313841fa9..d4a6c4b596 100644 --- a/cmd/cel-shed/p2p.go +++ b/cmd/cel-shed/p2p.go @@ -22,7 +22,7 @@ var p2pCmd = &cobra.Command{ var p2pNewKeyCmd = &cobra.Command{ Use: "new-key", Short: "Generate and print new Ed25519 private key for p2p networking", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { //nolint:revive privkey, _, err := crypto.GenerateEd25519Key(rand.Reader) if err != nil { return err @@ -42,7 +42,7 @@ var p2pNewKeyCmd = &cobra.Command{ var p2pPeerIDCmd = &cobra.Command{ Use: "peer-id", Short: "Get peer-id out of public or private Ed25519 key", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { //nolint:revive decKey, err := hex.DecodeString(args[0]) if err != nil { return err diff --git a/cmd/config.go b/cmd/config.go index 4a2d322adf..51bba81b45 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -12,7 +12,7 @@ func RemoveConfigCmd(fsets ...*flag.FlagSet) *cobra.Command { Use: "config-remove", Short: "Deletes the node's config", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() return nodebuilder.RemoveConfig(StorePath(ctx)) }, @@ -31,7 +31,7 @@ func UpdateConfigCmd(fsets ...*flag.FlagSet) *cobra.Command { Long: "Updates the node's outdated config with default values from newly-added fields. Check the config " + " afterwards to ensure all old custom values were preserved.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() return nodebuilder.UpdateConfig(NodeType(ctx), StorePath(ctx)) }, diff --git a/cmd/docgen/openrpc.go b/cmd/docgen/openrpc.go index cd5b57a82c..ef8b706ec5 100644 --- a/cmd/docgen/openrpc.go +++ b/cmd/docgen/openrpc.go @@ -14,7 +14,7 @@ import ( var rootCmd = &cobra.Command{ Use: "docgen [packages]", Short: "docgen generates the openrpc documentation for Celestia Node packages", - RunE: func(cmd *cobra.Command, moduleNames []string) error { + RunE: func(_ *cobra.Command, moduleNames []string) error { // 1. Open the respective nodebuilder/X/service.go files for AST parsing nodeComments, permComments := docgen.ParseCommentsFromNodebuilderModules(moduleNames...) diff --git a/cmd/flags_misc.go b/cmd/flags_misc.go index cd539bde4c..332d2f8022 100644 --- a/cmd/flags_misc.go +++ b/cmd/flags_misc.go @@ -7,8 +7,8 @@ import ( "net/http/pprof" "strings" + otelpyroscope "github.com/grafana/otel-profiling-go" logging "github.com/ipfs/go-log/v2" - otelpyroscope "github.com/pyroscope-io/otel-profiling-go" "github.com/spf13/cobra" flag "github.com/spf13/pflag" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" diff --git a/cmd/init.go b/cmd/init.go index 5eaa465701..abad602d94 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -13,7 +13,7 @@ func Init(fsets ...*flag.FlagSet) *cobra.Command { Use: "init", Short: "Initialization for Celestia Node. Passed flags have persisted effect.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() return nodebuilder.Init(NodeConfig(ctx), StorePath(ctx), NodeType(ctx)) diff --git a/cmd/reset_store.go b/cmd/reset_store.go index d386549efa..a839d765ab 100644 --- a/cmd/reset_store.go +++ b/cmd/reset_store.go @@ -13,7 +13,7 @@ func ResetStore(fsets ...*flag.FlagSet) *cobra.Command { Use: "unsafe-reset-store", Short: "Resets the node's store to a new state. Leaves the keystore and config intact.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() return nodebuilder.Reset(StorePath(ctx), NodeType(ctx)) diff --git a/cmd/start.go b/cmd/start.go index 281dfcc0e4..c3ee37fe72 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -25,7 +25,7 @@ Options passed on start override configuration options only on start and are not Aliases: []string{"run", "daemon"}, Args: cobra.NoArgs, SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) (err error) { + RunE: func(cmd *cobra.Command, _ []string) (err error) { ctx := cmd.Context() // override config with all modifiers passed on start diff --git a/das/doc.go b/das/doc.go index bc67fcc7f3..df60ae0d1c 100644 --- a/das/doc.go +++ b/das/doc.go @@ -2,7 +2,7 @@ Package das contains the most important functionality provided by celestia-node. It contains logic for running data availability sampling (DAS) routines on block headers in the network. DAS is the process of verifying the availability of -block data by sampling chunks or shares of those blocks. +block data by sampling shares of those blocks. Package das can confirm the availability of block data in the network via the Availability interface which is implemented both in `full` and `light` mode. diff --git a/docs/adr/adr-011-blocksync-overhaul-part-1.md b/docs/adr/adr-011-blocksync-overhaul-part-1.md index 2967f696ac..0173e78efb 100644 --- a/docs/adr/adr-011-blocksync-overhaul-part-1.md +++ b/docs/adr/adr-011-blocksync-overhaul-part-1.md @@ -94,7 +94,7 @@ are kept with additional components introduced. Altogether, existing and new com foundation of our improved block storage subsystem. The central data structure representing Celestia block data is EDS(`rsmt2d.ExtendedDataSquare`), and the new storage design -is focused around storing entire EDSes as a whole rather than a set of individual chunks, s.t. storage subsystem +is focused around storing entire EDSes as a whole rather than a set of individual shares, s.t. storage subsystem can handle storing and streaming/serving blocks of 4MB and more. #### EDS (De-)Serialization @@ -174,9 +174,9 @@ and getting data by namespace. ```go type Store struct { - basepath string + basepath string dgstr dagstore.DAGStore - topIdx index.Inverted + topIdx index.Inverted carIdx index.FullIndexRepo mounts *mount.Registry ... @@ -189,18 +189,18 @@ func NewStore(basepath string, ds datastore.Batching) *Store { carIdx := index.NewFSRepo(basepath + "/index") mounts := mount.NewRegistry() r := mount.NewRegistry() - err = r.Register("fs", &mount.FSMount{FS: os.DirFS(basePath + "/eds/")}) // registration is a must + err = r.Register("fs", &mount.FSMount{FS: os.DirFS(basePath + "/eds/")}) // registration is a must if err != nil { panic(err) } - + return &Store{ basepath: basepath, dgst: dagstore.New(dagstore.Config{...}), topIdx: index.NewInverted(datastore), carIdx: index.NewFSRepo(basepath + "/index") mounts: mounts, - } + } } ``` @@ -224,7 +224,7 @@ out of the scope of the document._ // // The square is verified on the Exchange level and Put only stores the square trusting it. // The resulting file stores all the shares and NMT Merkle Proofs of the EDS. -// Additionally, the file gets indexed s.t. Store.Blockstore can access them. +// Additionally, the file gets indexed s.t. Store.Blockstore can access them. func (s *Store) Put(context.Context, DataHash, *rsmt2d.ExtendedDataSquare) error ``` @@ -243,9 +243,9 @@ ___NOTES:___ ```go // GetCAR takes a DataHash and returns a buffered reader to the respective EDS serialized as a CARv1 file. -// -// The Reader strictly reads our full EDS, and it's integrity is not verified. -// +// +// The Reader strictly reads our full EDS, and it's integrity is not verified. +// // Caller must Close returned reader after reading. func (s *Store) GetCAR(context.Context, DataHash) (io.ReadCloser, error) ``` @@ -265,8 +265,8 @@ ___NOTES:___ - EDIT: We went with custom implementation. ```go -// Blockstore returns an IPFS Blockstore providing access to individual shares/nodes of all EDS -// registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. +// Blockstore returns an IPFS Blockstore providing access to individual shares/nodes of all EDS +// registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. // We represent `shares` and NMT Merkle proofs as IPFS blocks and IPLD nodes so Bitswap can access those. func (s *Store) Blockstore() blockstore.Blockstore ``` @@ -284,8 +284,8 @@ ___NOTES:___ blocks._ ```go -// CARBlockstore returns an IPFS Blockstore providing access to individual shares/nodes of a specific EDS identified by -// DataHash and registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. +// CARBlockstore returns an IPFS Blockstore providing access to individual shares/nodes of a specific EDS identified by +// DataHash and registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. // We represent `shares` and NMT Merkle proofs as IPFS blocks and IPLD nodes so Bitswap can access those. func (s *Store) CARBlockstore(context.Context, DataHash) (dagstore.ReadBlockstore, error) ``` @@ -301,7 +301,7 @@ The `GetDAH` method returns the DAH (`share.Root`) of the EDS identified by `Dat ```go // GetDAH returns the DataAvailabilityHeader for the EDS identified by DataHash. -func (s *Store) GetDAH(context.Context, share.DataHash) (*share.Root, error) +func (s *Store) GetDAH(context.Context, share.DataHash) (*share.Root, error) ``` ##### `eds.Store.Get` @@ -315,7 +315,7 @@ ___NOTE:___ _It's unnecessary, but an API ergonomics/symmetry nice-to-have._ ```go // Get reads EDS out of Store by given DataHash. -// +// // It reads only one quadrant(1/4) of the EDS and verifies the integrity of the stored data by recomputing it. func (s *Store) Get(context.Context, DataHash) (*rsmt2d.ExtendedDataSquare, error) ``` diff --git a/docs/adr/adr-012-daser-parallelization.md b/docs/adr/adr-012-daser-parallelization.md index a1f3af885b..95e70cee78 100644 --- a/docs/adr/adr-012-daser-parallelization.md +++ b/docs/adr/adr-012-daser-parallelization.md @@ -10,7 +10,7 @@ ## Context -DAS is the process of verifying the availability of block data by sampling chunks or shares of those blocks. The `das` package implements an engine to ensure the availability of the chain's block data via the `Availability` interface. +DAS is the process of verifying the availability of block data by sampling shares of those blocks. The `das` package implements an engine to ensure the availability of the chain's block data via the `Availability` interface. Verifying the availability of block data is a priority functionality for celestia-node. Its performance could benefit significantly from parallelization optimisation to make it able to fully utilise network bandwidth. ## Previous approach @@ -61,7 +61,7 @@ amount of workers: 32, speed: 11.33 amount of workers: 64, speed: 11.83 ``` -Based on basic experiment results, values higher than 16 don’t bring much benefit. At the same time, increased parallelization comes with a cost of higher memory consumption. +Based on basic experiment results, values higher than 16 don’t bring much benefit. At the same time, increased parallelization comes with a cost of higher memory consumption. Future improvements will be discussed later and are out of the scope of this ADR. ## Status @@ -70,7 +70,7 @@ Implemented ## Future plans -Several params values that come hardcoded in DASer (`samplingRange`, `concurrencyLimit`, `priorityQueueSize`, `genesisHeight`, `backgroundStoreInterval`) should become configurable, so the node runner can define them based on the specific node setup. Default values should be optimized by performance testing for most common setups, and could potentially vary for different node types. +Several params values that come hardcoded in DASer (`samplingRange`, `concurrencyLimit`, `priorityQueueSize`, `genesisHeight`, `backgroundStoreInterval`) should become configurable, so the node runner can define them based on the specific node setup. Default values should be optimized by performance testing for most common setups, and could potentially vary for different node types. ## References diff --git a/go.mod b/go.mod index da96f82757..564133553e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/celestiaorg/celestia-node -go 1.21.1 +go 1.22.0 require ( cosmossdk.io/errors v1.0.1 @@ -8,13 +8,13 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.4.0 + github.com/celestiaorg/celestia-app v1.7.0 github.com/celestiaorg/go-fraud v0.2.0 - github.com/celestiaorg/go-header v0.5.3 + github.com/celestiaorg/go-header v0.5.4 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.20.0 github.com/celestiaorg/rsmt2d v0.11.0 - github.com/cosmos/cosmos-sdk v0.46.14 + github.com/cosmos/cosmos-sdk v0.46.16 github.com/cosmos/cosmos-sdk/api v0.1.0 github.com/cristalhq/jwt v1.2.0 github.com/dgraph-io/badger/v4 v4.2.1-0.20240106094458-1c417aa3799c @@ -22,9 +22,12 @@ require ( github.com/filecoin-project/dagstore v0.5.6 github.com/filecoin-project/go-jsonrpc v0.3.1 github.com/gammazero/workerpool v1.1.3 + github.com/gofrs/flock v0.8.1 github.com/gogo/protobuf v1.3.3 github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.1 + github.com/grafana/otel-profiling-go v0.5.1 + github.com/grafana/pyroscope-go v1.1.1 github.com/hashicorp/go-retryablehttp v0.7.5 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/imdario/mergo v0.3.16 @@ -50,13 +53,11 @@ require ( github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/prometheus/client_golang v1.18.0 - github.com/pyroscope-io/client v0.7.2 - github.com/pyroscope-io/otel-profiling-go v0.5.0 github.com/rollkit/go-da v0.4.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.4 - github.com/tendermint/tendermint v0.34.28 + github.com/stretchr/testify v1.9.0 + github.com/tendermint/tendermint v0.34.29 go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.24.0 @@ -68,8 +69,8 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 go.opentelemetry.io/proto/otlp v1.1.0 go.uber.org/fx v1.20.1 - go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.19.0 + go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.21.0 golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 @@ -137,10 +138,10 @@ require ( github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/dvsekhvalnov/jose2go v1.5.0 // indirect + github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect - github.com/ethereum/c-kzg-4844 v0.3.1 // indirect + github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/ethereum/go-ethereum v1.13.2 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flynn/noise v1.0.1 // indirect @@ -149,7 +150,7 @@ require ( github.com/gammazero/deque v0.2.0 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -178,6 +179,7 @@ require ( github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect @@ -239,7 +241,7 @@ require ( github.com/libp2p/go-netroute v0.2.1 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.1 // indirect - github.com/magiconair/properties v1.8.6 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/manifoldco/promptui v0.9.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect @@ -267,7 +269,6 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect @@ -277,7 +278,6 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/pyroscope-io/godeltaprof v0.1.2 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-20 v0.4.1 // indirect github.com/quic-go/quic-go v0.40.1 // indirect @@ -286,17 +286,16 @@ require ( github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/regen-network/cosmos-proto v0.3.1 // indirect - github.com/rivo/uniseg v0.4.4 // indirect - github.com/rs/cors v1.8.2 // indirect + github.com/rs/cors v1.8.3 // indirect github.com/rs/zerolog v1.31.0 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/shirou/gopsutil v3.21.6+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.9.2 // indirect + github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/viper v1.14.0 // indirect - github.com/subosito/gotenv v1.4.1 // indirect + github.com/spf13/viper v1.15.0 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/supranational/blst v0.3.11 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect @@ -321,10 +320,10 @@ require ( go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.20.0 // indirect + golang.org/x/net v0.21.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.16.1 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect @@ -344,10 +343,10 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.20.1-sdk-v0.46.16 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.35.0-tm-v0.34.29 ) diff --git a/go.sum b/go.sum index a5ea60f868..b062866923 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOv github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -335,8 +335,8 @@ github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJ github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= -github.com/btcsuite/btcd/btcutil v1.1.2 h1:XLMbX8JQEiwMcYft2EGi8zPUkoa0abKIU6/BJSRsjzQ= -github.com/btcsuite/btcd/btcutil v1.1.2/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= +github.com/btcsuite/btcd/btcutil v1.1.3 h1:xfbtw8lwpp0G6NwSHb+UE67ryTFHJAiNuipusjXSohQ= +github.com/btcsuite/btcd/btcutil v1.1.3/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= @@ -352,24 +352,24 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/bufbuild/protocompile v0.1.0 h1:HjgJBI85hY/qmW5tw/66sNDZ7z0UDdVSi/5r40WHw4s= -github.com/bufbuild/protocompile v0.1.0/go.mod h1:ix/MMMdsT3fzxfw91dvbfzKW3fRRnuPCP47kpAm5m/4= +github.com/bufbuild/protocompile v0.5.1 h1:mixz5lJX4Hiz4FpqFREJHIXLfaLBntfaJv1h+/jS+Qg= +github.com/bufbuild/protocompile v0.5.1/go.mod h1:G5iLmavmF4NsYtpZFvE3B/zFch2GIY8+wjsYLR/lc40= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.4.0 h1:hTId3xL8GssN5sHSHPP7svHi/iWp+XVxphStiR7ADiY= -github.com/celestiaorg/celestia-app v1.4.0/go.mod h1:zhdQIFGFZRRxrDVtFE4OFIT7/12RE8DRyfvNZdW8ceM= -github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 h1:Fd7ymPUzExPGNl2gZw4i5S74arMw+iDHLE78M/cCxl4= -github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29/go.mod h1:xrICN0PBhp3AdTaZ8q4wS5Jvi32V02HNjaC2EsWiEKk= -github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14 h1:+Te28r5Zp4Vp69f82kcON9/BIF8f1BNXb0go2+achuc= -github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14/go.mod h1:Og5KKgoBEiQlI6u56lDLG191pfknIdXctFn3COWLQP8= +github.com/celestiaorg/celestia-app v1.7.0 h1:pFIgQzeMD036ulJ2ShUR/3bDGG6kwNAWaLEE0MXHmgg= +github.com/celestiaorg/celestia-app v1.7.0/go.mod h1:z2H47Gs9gYd3GdQ22d5sbcL8/aBMRcVDtUWT64goMaY= +github.com/celestiaorg/celestia-core v1.35.0-tm-v0.34.29 h1:sXERzNXgyHyqTKNQx4S29C/NMDzgav62DaQDNF49HUQ= +github.com/celestiaorg/celestia-core v1.35.0-tm-v0.34.29/go.mod h1:weZR4wYx1Vcw3g1Jc5G8VipG4M+KUDSqeIzyyWszmsQ= +github.com/celestiaorg/cosmos-sdk v1.20.1-sdk-v0.46.16 h1:9U9UthIJSOyVjabD5PkD6aczvqlWOyAFTOXw0duPT5k= +github.com/celestiaorg/cosmos-sdk v1.20.1-sdk-v0.46.16/go.mod h1:Tvsc3YnqvflXTYC8xIy/Q07Es95xZ1pZC/imoKogtbg= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJx5/hgZ+IeOLEIoLsAveJN/7/ZtQQtPSVw= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= -github.com/celestiaorg/go-header v0.5.3 h1:8CcflT6aIlcQXKNWcMekoBNs3EU50mEmDp17gbn1pP4= -github.com/celestiaorg/go-header v0.5.3/go.mod h1:7BVR6myjRfACbqW1de6s8OjuK66XzHm8MpFNYr0G+nU= +github.com/celestiaorg/go-header v0.5.4 h1:7okm3N17/kE5VanlIHv7+bWh5TNAkJxVoNULsYnphAM= +github.com/celestiaorg/go-header v0.5.4/go.mod h1:7BVR6myjRfACbqW1de6s8OjuK66XzHm8MpFNYr0G+nU= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= @@ -579,8 +579,8 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:Htrtb github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM= -github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= +github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= +github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -606,8 +606,8 @@ github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakR github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= github.com/etclabscore/go-openrpc-reflect v0.0.37 h1:IH0e7JqIvR9OhbbFWi/BHIkXrqbR3Zyia3RJ733eT6c= github.com/etclabscore/go-openrpc-reflect v0.0.37/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= -github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= -github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= +github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= github.com/ethereum/go-ethereum v1.13.2 h1:g9mCpfPWqCA1OL4e6C98PeVttb0HadfBRuKTGvMnOvw= github.com/ethereum/go-ethereum v1.13.2/go.mod h1:gkQ5Ygi64ZBh9M/4iXY1R8WqoNCx1Ey0CkYn2BD4/fw= @@ -618,8 +618,9 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqL github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -688,8 +689,9 @@ github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBj github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -918,6 +920,12 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8= +github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls= +github.com/grafana/pyroscope-go v1.1.1 h1:PQoUU9oWtO3ve/fgIiklYuGilvsm8qaGhlY4Vw6MAcQ= +github.com/grafana/pyroscope-go v1.1.1/go.mod h1:Mw26jU7jsL/KStNSGGuuVYdUq7Qghem5P8aXYXSXG88= +github.com/grafana/pyroscope-go/godeltaprof v0.1.6 h1:nEdZ8louGAplSvIJi1HVp7kWvFvdiiYg3COLlTwJiFo= +github.com/grafana/pyroscope-go/godeltaprof v0.1.6/go.mod h1:Tk376Nbldo4Cha9RgiU7ik8WKFkNpfds98aUzS8omLE= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -973,8 +981,9 @@ github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoD github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -1341,6 +1350,7 @@ github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -1681,8 +1691,8 @@ github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -1998,8 +2008,6 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= @@ -2085,12 +2093,6 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/pyroscope-io/client v0.7.2 h1:OX2qdUQsS8RSkn/3C8isD7f/P0YiZQlRbAlecAaj/R8= -github.com/pyroscope-io/client v0.7.2/go.mod h1:FEocnjn+Ngzxy6EtU9ZxXWRvQ0+pffkrBxHLnPpxwi8= -github.com/pyroscope-io/godeltaprof v0.1.2 h1:MdlEmYELd5w+lvIzmZvXGNMVzW2Qc9jDMuJaPOR75g4= -github.com/pyroscope-io/godeltaprof v0.1.2/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= -github.com/pyroscope-io/otel-profiling-go v0.5.0 h1:LsTP9VuQ5TgeSiyY2gPHy1de/q3jbFyGWE1v3LtHzMk= -github.com/pyroscope-io/otel-profiling-go v0.5.0/go.mod h1:jUUUXTTgntvGJKS8p5uzypXwTyuGnQP31VnWauH/lUg= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= @@ -2131,8 +2133,8 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/rollkit/go-da v0.4.0 h1:/s7ZrVq7DC2aK8UXIvB7rsXrZ2mVGRw7zrexcxRvhlw= github.com/rollkit/go-da v0.4.0/go.mod h1:Kef0XI5ecEKd3TXzI8S+9knAUJnZg0svh2DuXoCsPlM= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= -github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= +github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= @@ -2206,8 +2208,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= -github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= @@ -2223,8 +2225,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= -github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= @@ -2235,8 +2237,9 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -2249,10 +2252,11 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= -github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -2399,7 +2403,7 @@ go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0/go.mod h1:ch3a5QxOqV go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= -go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.24.0 h1:mM8nKi6/iFQ0iqst80wDHU2ge198Ye/TfN0WBS5U24Y= @@ -2411,11 +2415,12 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 h1:VhlEQAPp9R1ktYfrPk5SOryw1e9LDDTZCbIPFrho0ec= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0/go.mod h1:kB3ufRbfU+CQ4MlUcqtW8Z7YEOBeK2DJ6CmR5rYYF3E= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8= @@ -2423,7 +2428,7 @@ go.opentelemetry.io/otel/sdk/metric v1.24.0/go.mod h1:I6Y5FjH6rvEnTTAYQz3Mmv2kl6 go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= -go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -2449,8 +2454,8 @@ go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -2474,8 +2479,8 @@ go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -2524,8 +2529,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2678,8 +2683,8 @@ golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2872,8 +2877,9 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2882,8 +2888,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/header/headertest/testing.go b/header/headertest/testing.go index e97f7f7825..7b0ae64262 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -36,19 +36,24 @@ type TestSuite struct { valPntr int head *header.ExtendedHeader + + // blockTime is optional - if set, the test suite will generate + // blocks timestamped at the specified interval + blockTime time.Duration } func NewStore(t *testing.T) libhead.Store[*header.ExtendedHeader] { - return headertest.NewStore[*header.ExtendedHeader](t, NewTestSuite(t, 3), 10) + return headertest.NewStore[*header.ExtendedHeader](t, NewTestSuite(t, 3, 0), 10) } // NewTestSuite setups a new test suite with a given number of validators. -func NewTestSuite(t *testing.T, num int) *TestSuite { - valSet, vals := RandValidatorSet(num, 10) +func NewTestSuite(t *testing.T, numValidators int, blockTime time.Duration) *TestSuite { + valSet, vals := RandValidatorSet(numValidators, 10) return &TestSuite{ - t: t, - vals: vals, - valSet: valSet, + t: t, + vals: vals, + valSet: valSet, + blockTime: blockTime, } } @@ -155,13 +160,18 @@ func (s *TestSuite) GenRawHeader( height uint64, lastHeader, lastCommit, dataHash libhead.Hash) *header.RawHeader { rh := RandRawHeader(s.t) rh.Height = int64(height) - rh.Time = time.Now() rh.LastBlockID = types.BlockID{Hash: bytes.HexBytes(lastHeader)} rh.LastCommitHash = bytes.HexBytes(lastCommit) rh.DataHash = bytes.HexBytes(dataHash) rh.ValidatorsHash = s.valSet.Hash() rh.NextValidatorsHash = s.valSet.Hash() rh.ProposerAddress = s.nextProposer().Address + + rh.Time = time.Now() + if s.blockTime > 0 { + rh.Time = s.Head().Time().Add(s.blockTime) + } + return rh } diff --git a/header/headertest/verify_test.go b/header/headertest/verify_test.go index 82795ca5ff..c55d810593 100644 --- a/header/headertest/verify_test.go +++ b/header/headertest/verify_test.go @@ -12,7 +12,7 @@ import ( ) func TestVerify(t *testing.T) { - h := NewTestSuite(t, 2).GenExtendedHeaders(3) + h := NewTestSuite(t, 2, 0).GenExtendedHeaders(3) trusted, untrustedAdj, untrustedNonAdj := h[0], h[1], h[2] tests := []struct { prepare func() *header.ExtendedHeader @@ -47,7 +47,7 @@ func TestVerify(t *testing.T) { { prepare: func() *header.ExtendedHeader { untrusted := *untrustedNonAdj - untrusted.Commit = NewTestSuite(t, 2).Commit(RandRawHeader(t)) + untrusted.Commit = NewTestSuite(t, 2, 0).Commit(RandRawHeader(t)) return &untrusted }, err: header.ErrVerifyCommitLightTrustingFailed, diff --git a/libs/fslock/lock_unix.go b/libs/fslock/lock_unix.go deleted file mode 100644 index bbb8db8537..0000000000 --- a/libs/fslock/lock_unix.go +++ /dev/null @@ -1,48 +0,0 @@ -//go:build darwin || freebsd || linux - -package fslock - -import ( - "fmt" - "os" - "strconv" - "syscall" -) - -func (l *Locker) lock() (err error) { - l.file, err = os.OpenFile(l.path, os.O_CREATE|os.O_RDWR, 0666) - if err != nil { - return fmt.Errorf("fslock: error opening file: %w", err) - } - - _, err = l.file.WriteString(strconv.Itoa(os.Getpid())) - if err != nil { - return fmt.Errorf("fslock: error writing process id: %w", err) - } - - err = syscall.Flock(int(l.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) - if err != nil && err.Error() == "resource temporarily unavailable" { - return ErrLocked // we have to check here for a string in err, as there is no error types defined for this case. - } - if err != nil { - return fmt.Errorf("fslock: flocking error: %w", err) - } - - return -} - -func (l *Locker) unlock() error { - err := syscall.Flock(int(l.file.Fd()), syscall.LOCK_UN|syscall.LOCK_NB) - if err != nil { - return fmt.Errorf("fslock: unflocking error: %w", err) - } - - file := l.file - l.file = nil - err = file.Close() - if err != nil { - return fmt.Errorf("fslock: while closing file: %w", err) - } - - return os.Remove(l.path) -} diff --git a/libs/fslock/locker.go b/libs/fslock/locker.go deleted file mode 100644 index f451c42cc1..0000000000 --- a/libs/fslock/locker.go +++ /dev/null @@ -1,49 +0,0 @@ -package fslock - -import ( - "errors" - "os" -) - -// ErrLocked is signaled when someone tries to lock an already locked file. -var ErrLocked = errors.New("fslock: directory is locked") - -// Lock creates a new Locker under the given 'path' -// and immediately does Lock on it. -func Lock(path string) (*Locker, error) { - l := New(path) - err := l.Lock() - if err != nil { - return nil, err - } - - return l, nil -} - -// Locker is a simple utility meant to create lock files. -// This is to prevent multiple processes from managing the same working directory by purpose or -// accident. NOTE: Windows is not supported. -type Locker struct { - file *os.File - path string -} - -// New creates a new Locker with a File pointing to the given 'path'. -func New(path string) *Locker { - return &Locker{path: path} -} - -// Lock locks the file. -// Subsequent calls will error with ErrLocked on any Locker instance looking to the same path. -func (l *Locker) Lock() error { - return l.lock() -} - -// Unlock frees up the lock. -func (l *Locker) Unlock() error { - if l == nil || l.file == nil { - return nil - } - - return l.unlock() -} diff --git a/libs/fslock/locker_test.go b/libs/fslock/locker_test.go deleted file mode 100644 index c80ce51c16..0000000000 --- a/libs/fslock/locker_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package fslock - -import ( - "os" - "path/filepath" - "testing" -) - -func TestLocker(t *testing.T) { - path := filepath.Join(os.TempDir(), ".lock") - defer os.Remove(path) - - locker := New(path) - locker2 := New(path) - - err := locker.Lock() - if err != nil { - t.Fatal(err) - } - - err = locker2.Lock() - if err != ErrLocked { - t.Fatal("No locking") - } - - err = locker.Unlock() - if err != nil { - t.Fatal(err) - } - - err = locker2.Lock() - if err != nil { - t.Fatal(err) - } - - err = locker2.Unlock() - if err != nil { - t.Fatal(err) - } -} diff --git a/nodebuilder/blob/cmd/blob.go b/nodebuilder/blob/cmd/blob.go index 25a102843b..78dd57f250 100644 --- a/nodebuilder/blob/cmd/blob.go +++ b/nodebuilder/blob/cmd/blob.go @@ -272,6 +272,7 @@ func formatData(data interface{}) interface{} { Data string `json:"data"` ShareVersion uint32 `json:"share_version"` Commitment []byte `json:"commitment"` + Index int `json:"index"` } if reflect.TypeOf(data).Kind() == reflect.Slice { @@ -283,6 +284,7 @@ func formatData(data interface{}) interface{} { Data: string(b.Data), ShareVersion: b.ShareVersion, Commitment: b.Commitment, + Index: b.Index(), } } return result @@ -294,5 +296,6 @@ func formatData(data interface{}) interface{} { Data: string(b.Data), ShareVersion: b.ShareVersion, Commitment: b.Commitment, + Index: b.Index(), } } diff --git a/nodebuilder/blob/module.go b/nodebuilder/blob/module.go index 76e7677725..56a6c906e9 100644 --- a/nodebuilder/blob/module.go +++ b/nodebuilder/blob/module.go @@ -8,8 +8,8 @@ import ( "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/header" headerService "github.com/celestiaorg/celestia-node/nodebuilder/header" + "github.com/celestiaorg/celestia-node/nodebuilder/state" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/state" ) func ConstructModule() fx.Option { @@ -19,7 +19,7 @@ func ConstructModule() fx.Option { return service.GetByHeight }), fx.Provide(func( - state *state.CoreAccessor, + state state.Module, sGetter share.Getter, getByHeightFn func(context.Context, uint64) (*header.ExtendedHeader, error), ) Module { diff --git a/nodebuilder/config.go b/nodebuilder/config.go index 41f24d6d3d..d323f401d7 100644 --- a/nodebuilder/config.go +++ b/nodebuilder/config.go @@ -1,13 +1,14 @@ package nodebuilder import ( + "fmt" "io" "os" "github.com/BurntSushi/toml" + "github.com/gofrs/flock" "github.com/imdario/mergo" - "github.com/celestiaorg/celestia-node/libs/fslock" "github.com/celestiaorg/celestia-node/nodebuilder/core" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/gateway" @@ -91,14 +92,15 @@ func RemoveConfig(path string) (err error) { return } - flock, err := fslock.Lock(lockPath(path)) + flk := flock.New(lockPath(path)) + ok, err := flk.TryLock() if err != nil { - if err == fslock.ErrLocked { - err = ErrOpened - } - return + return fmt.Errorf("locking file: %w", err) + } + if !ok { + return ErrOpened } - defer flock.Unlock() //nolint: errcheck + defer flk.Unlock() //nolint:errcheck return removeConfig(configPath(path)) } @@ -117,14 +119,15 @@ func UpdateConfig(tp node.Type, path string) (err error) { return err } - flock, err := fslock.Lock(lockPath(path)) + flk := flock.New(lockPath(path)) + ok, err := flk.TryLock() if err != nil { - if err == fslock.ErrLocked { - err = ErrOpened - } - return err + return fmt.Errorf("locking file: %w", err) + } + if !ok { + return ErrOpened } - defer flock.Unlock() //nolint: errcheck + defer flk.Unlock() //nolint:errcheck newCfg := DefaultConfig(tp) diff --git a/nodebuilder/core/module.go b/nodebuilder/core/module.go index 7c5c9e6bfd..fcf682cdf5 100644 --- a/nodebuilder/core/module.go +++ b/nodebuilder/core/module.go @@ -74,10 +74,10 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option )), fx.Provide(fx.Annotate( remote, - fx.OnStart(func(ctx context.Context, client core.Client) error { + fx.OnStart(func(_ context.Context, client core.Client) error { return client.Start() }), - fx.OnStop(func(ctx context.Context, client core.Client) error { + fx.OnStop(func(_ context.Context, client core.Client) error { return client.Stop() }), )), diff --git a/nodebuilder/da/service.go b/nodebuilder/da/service.go index b775e10396..ea435a5c36 100644 --- a/nodebuilder/da/service.go +++ b/nodebuilder/da/service.go @@ -3,6 +3,7 @@ package da import ( "context" "encoding/binary" + "encoding/json" "fmt" "strings" @@ -83,7 +84,7 @@ func (s *Service) GetProofs(ctx context.Context, ids []da.ID, namespace da.Names if err != nil { return nil, err } - proofs[i], err = proof.MarshalJSON() + proofs[i], err = json.Marshal(proof) if err != nil { return nil, err } @@ -153,7 +154,7 @@ func (s *Service) Validate( proofs := make([]*blob.Proof, len(ids)) for i, daProof := range daProofs { blobProof := &blob.Proof{} - err := blobProof.UnmarshalJSON(daProof) + err := json.Unmarshal(daProof, blobProof) if err != nil { return nil, err } diff --git a/nodebuilder/das/cmd/das.go b/nodebuilder/das/cmd/das.go index 7512861ac3..5678190c9c 100644 --- a/nodebuilder/das/cmd/das.go +++ b/nodebuilder/das/cmd/das.go @@ -21,7 +21,7 @@ var samplingStatsCmd = &cobra.Command{ Use: "sampling-stats", Short: "Returns the current statistics over the DA sampling process", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err diff --git a/nodebuilder/header/cmd/header.go b/nodebuilder/header/cmd/header.go index b3bba1eb32..6bcf84d414 100644 --- a/nodebuilder/header/cmd/header.go +++ b/nodebuilder/header/cmd/header.go @@ -31,7 +31,7 @@ var localHeadCmd = &cobra.Command{ Use: "local-head", Short: "Returns the ExtendedHeader from the chain head.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -47,7 +47,7 @@ var networkHeadCmd = &cobra.Command{ Use: "network-head", Short: "Provides the Syncer's view of the current network head.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -104,7 +104,7 @@ var syncStateCmd = &cobra.Command{ Use: "sync-state", Short: "Returns the current state of the header Syncer.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err diff --git a/nodebuilder/init.go b/nodebuilder/init.go index 0593d88560..fb3aeedf02 100644 --- a/nodebuilder/init.go +++ b/nodebuilder/init.go @@ -8,11 +8,11 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/hd" "github.com/cosmos/cosmos-sdk/crypto/keyring" sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/gofrs/flock" "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" - "github.com/celestiaorg/celestia-node/libs/fslock" "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/state" @@ -35,14 +35,15 @@ func Init(cfg Config, path string, tp node.Type) error { return err } - flock, err := fslock.Lock(lockPath(path)) + flk := flock.New(lockPath(path)) + ok, err := flk.TryLock() if err != nil { - if err == fslock.ErrLocked { - return ErrOpened - } - return err + return fmt.Errorf("locking file: %w", err) + } + if !ok { + return ErrOpened } - defer flock.Unlock() //nolint: errcheck + defer flk.Unlock() //nolint:errcheck ksPath := keysPath(path) err = initDir(ksPath) @@ -82,14 +83,15 @@ func Reset(path string, tp node.Type) error { } log.Infof("Resetting %s Node Store over '%s'", tp, path) - flock, err := fslock.Lock(lockPath(path)) + flk := flock.New(lockPath(path)) + ok, err := flk.TryLock() if err != nil { - if err == fslock.ErrLocked { - return ErrOpened - } - return err + return fmt.Errorf("locking file: %w", err) + } + if !ok { + return ErrOpened } - defer flock.Unlock() //nolint: errcheck + defer flk.Unlock() //nolint:errcheck err = resetDir(dataPath(path)) if err != nil { diff --git a/nodebuilder/init_test.go b/nodebuilder/init_test.go index e438a191bc..dcac1466af 100644 --- a/nodebuilder/init_test.go +++ b/nodebuilder/init_test.go @@ -7,13 +7,13 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/hd" "github.com/cosmos/cosmos-sdk/crypto/keyring" sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/gofrs/flock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" - "github.com/celestiaorg/celestia-node/libs/fslock" "github.com/celestiaorg/celestia-node/nodebuilder/node" ) @@ -58,9 +58,10 @@ func TestIsInitForNonExistDir(t *testing.T) { func TestInitErrForLockedDir(t *testing.T) { dir := t.TempDir() - flock, err := fslock.Lock(lockPath(dir)) + flk := flock.New(lockPath(dir)) + _, err := flk.TryLock() require.NoError(t, err) - defer flock.Unlock() //nolint:errcheck + defer flk.Unlock() //nolint:errcheck nodes := []node.Type{node.Light, node.Bridge} for _, node := range nodes { diff --git a/nodebuilder/node/cmd/node.go b/nodebuilder/node/cmd/node.go index a65727fb03..bef76b9683 100644 --- a/nodebuilder/node/cmd/node.go +++ b/nodebuilder/node/cmd/node.go @@ -25,7 +25,7 @@ var nodeInfoCmd = &cobra.Command{ Use: "info", Args: cobra.NoArgs, Short: "Returns administrative information about the node.", - RunE: func(c *cobra.Command, args []string) error { + RunE: func(c *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(c.Context()) if err != nil { return err diff --git a/nodebuilder/node/metrics.go b/nodebuilder/node/metrics.go index 07c9a5fc0f..560df808e6 100644 --- a/nodebuilder/node/metrics.go +++ b/nodebuilder/node/metrics.go @@ -42,7 +42,7 @@ func WithMetrics() error { return err } - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { if !nodeStarted { // Observe node start timestamp timeStarted = time.Now() diff --git a/nodebuilder/p2p/bitswap.go b/nodebuilder/p2p/bitswap.go index 014435071a..773d39ba4e 100644 --- a/nodebuilder/p2p/bitswap.go +++ b/nodebuilder/p2p/bitswap.go @@ -53,7 +53,7 @@ func dataExchange(params bitSwapParams) exchange.Interface { net.Start(srvr, clnt) // starting with hook does not work params.Lifecycle.Append(fx.Hook{ - OnStop: func(ctx context.Context) (err error) { + OnStop: func(_ context.Context) (err error) { err = errors.Join(err, clnt.Close()) err = errors.Join(err, srvr.Close()) net.Stop() diff --git a/nodebuilder/p2p/cmd/p2p.go b/nodebuilder/p2p/cmd/p2p.go index 8b44802947..64c36fc9d6 100644 --- a/nodebuilder/p2p/cmd/p2p.go +++ b/nodebuilder/p2p/cmd/p2p.go @@ -48,7 +48,7 @@ var infoCmd = &cobra.Command{ Use: "info", Short: "Gets the node's peer info (peer id and multiaddresses)", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -77,7 +77,7 @@ var peersCmd = &cobra.Command{ Use: "peers", Short: "Lists the peers we are connected to", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -226,7 +226,7 @@ var natStatusCmd = &cobra.Command{ Use: "nat-status", Short: "Gets the current NAT status", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -328,7 +328,7 @@ var blockedPeersCmd = &cobra.Command{ Use: "blocked-peers", Short: "Lists the node's blocked peers", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -466,7 +466,7 @@ var bandwidthStatsCmd = &cobra.Command{ Long: "Get stats struct with bandwidth metrics for all data sent/" + "received by the local peer, regardless of protocol or remote peer IDs", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err diff --git a/nodebuilder/p2p/metrics.go b/nodebuilder/p2p/metrics.go index 095c30d9b7..ca226c98ae 100644 --- a/nodebuilder/p2p/metrics.go +++ b/nodebuilder/p2p/metrics.go @@ -41,7 +41,7 @@ func prometheusMetrics(lifecycle fx.Lifecycle, registerer prometheus.Registerer) } lifecycle.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { go func() { if err := promHTTPServer.ListenAndServe(); err != nil { log.Errorf("Error starting Prometheus metrics exporter http server: %s", err) diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 298976fda4..7830f0e8f6 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -5,10 +5,10 @@ import ( "fmt" "time" + otelpyroscope "github.com/grafana/otel-profiling-go" + "github.com/grafana/pyroscope-go" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/peer" - "github.com/pyroscope-io/client/pyroscope" - otelpyroscope "github.com/pyroscope-io/otel-profiling-go" "go.opentelemetry.io/contrib/instrumentation/runtime" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index 7caaf39a92..b81149535f 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -100,7 +100,7 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option func discoveryComponents(cfg *Config) fx.Option { return fx.Options( - fx.Invoke(func(disc *disc.Discovery) {}), + fx.Invoke(func(_ *disc.Discovery) {}), fx.Provide(fx.Annotate( newDiscovery(cfg.Discovery), fx.OnStart(func(ctx context.Context, d *disc.Discovery) error { @@ -189,7 +189,7 @@ func shrexGetterComponents(cfg *Config) fx.Option { func shrexServerComponents(cfg *Config) fx.Option { return fx.Options( - fx.Invoke(func(edsSrv *shrexeds.Server, ndSrc *shrexnd.Server) {}), + fx.Invoke(func(_ *shrexeds.Server, _ *shrexnd.Server) {}), fx.Provide(fx.Annotate( func(host host.Host, store *eds.Store, network modp2p.Network) (*shrexeds.Server, error) { cfg.ShrExEDSParams.WithNetworkID(network.String()) diff --git a/nodebuilder/state/cmd/state.go b/nodebuilder/state/cmd/state.go index d35c4a1b4f..031c118f98 100644 --- a/nodebuilder/state/cmd/state.go +++ b/nodebuilder/state/cmd/state.go @@ -40,7 +40,7 @@ var accountAddressCmd = &cobra.Command{ Use: "account-address", Short: "Retrieves the address of the node's account/signer.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -57,7 +57,7 @@ var balanceCmd = &cobra.Command{ Short: "Retrieves the Celestia coin balance for the node's account/signer and verifies it against " + "the corresponding block's AppHash.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err diff --git a/nodebuilder/store.go b/nodebuilder/store.go index 7f67a9e782..ba6c19eaa2 100644 --- a/nodebuilder/store.go +++ b/nodebuilder/store.go @@ -10,11 +10,11 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/keyring" "github.com/dgraph-io/badger/v4/options" + "github.com/gofrs/flock" "github.com/ipfs/go-datastore" dsbadger "github.com/ipfs/go-ds-badger4" "github.com/mitchellh/go-homedir" - "github.com/celestiaorg/celestia-node/libs/fslock" "github.com/celestiaorg/celestia-node/libs/keystore" "github.com/celestiaorg/celestia-node/share" ) @@ -58,28 +58,29 @@ func OpenStore(path string, ring keyring.Keyring) (Store, error) { return nil, err } - flock, err := fslock.Lock(lockPath(path)) + flk := flock.New(lockPath(path)) + ok, err := flk.TryLock() if err != nil { - if err == fslock.ErrLocked { - return nil, ErrOpened - } - return nil, err + return nil, fmt.Errorf("locking file: %w", err) } - - ok := IsInit(path) if !ok { - flock.Unlock() //nolint:errcheck - return nil, ErrNotInited + return nil, ErrOpened + } + + if !IsInit(path) { + err := errors.Join(ErrNotInited, flk.Unlock()) + return nil, err } ks, err := keystore.NewFSKeystore(keysPath(path), ring) if err != nil { + err = errors.Join(err, flk.Unlock()) return nil, err } return &fsStore{ path: path, - dirLock: flock, + dirLock: flk, keys: ks, }, nil } @@ -131,7 +132,7 @@ func (f *fsStore) Datastore() (datastore.Batching, error) { } func (f *fsStore) Close() (err error) { - err = errors.Join(err, f.dirLock.Unlock()) + err = errors.Join(err, f.dirLock.Close()) f.dataMu.Lock() if f.data != nil { err = errors.Join(err, f.data.Close()) @@ -146,7 +147,7 @@ type fsStore struct { dataMu sync.Mutex data datastore.Batching keys keystore.Keystore - dirLock *fslock.Locker // protects directory + dirLock *flock.Flock // protects directory } func storePath(path string) (string, error) { @@ -158,7 +159,7 @@ func configPath(base string) string { } func lockPath(base string) string { - return filepath.Join(base, "lock") + return filepath.Join(base, ".lock") } func keysPath(base string) string { diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index d0aeefd568..7eb225a14a 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -79,7 +79,7 @@ func TestBlobModule(t *testing.T) { time.Sleep(time.Second) blob1, err := fullClient.Blob.Get(ctx, height, blobs[0].Namespace(), blobs[0].Commitment) require.NoError(t, err) - require.Equal(t, blobs[0], blob1) + require.Equal(t, blobs[0].Commitment, blob1.Commitment) }, }, { @@ -151,7 +151,7 @@ func TestBlobModule(t *testing.T) { b0, err := fullClient.Blob.Get(ctx, height, b.Namespace(), b.Commitment) require.NoError(t, err) - require.Equal(t, b, b0) + require.Equal(t, b.Commitment, b0.Commitment) // give some time to store the data, // otherwise the test will hang on the IPLD level. @@ -180,7 +180,7 @@ func TestBlobModule(t *testing.T) { b0, err := fullClient.Blob.Get(ctx, h, blobs[0].Namespace(), blobs[0].Commitment) require.NoError(t, err) - require.Equal(t, blobs[0], b0) + require.Equal(t, blobs[0].Commitment, b0.Commitment) // give some time to store the data, // otherwise the test will hang on the IPLD level. diff --git a/share/availability/light/availability.go b/share/availability/light/availability.go index 1d35542344..97046f4438 100644 --- a/share/availability/light/availability.go +++ b/share/availability/light/availability.go @@ -8,7 +8,6 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/autobatch" "github.com/ipfs/go-datastore/namespace" - ipldFormat "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" "github.com/celestiaorg/celestia-node/header" @@ -67,26 +66,37 @@ func (la *ShareAvailability) SharesAvailable(ctx context.Context, header *header return nil } - // do not sample over Root that has already been sampled + // load snapshot of the last sampling errors from disk key := rootKey(dah) - la.dsLk.RLock() - exists, err := la.ds.Has(ctx, key) + last, err := la.ds.Get(ctx, key) la.dsLk.RUnlock() - if err != nil || exists { + + // Check for error cases + var samples []Sample + switch { + case err == nil && len(last) == 0: + // Availability has already been validated + return nil + case err != nil && !errors.Is(err, datastore.ErrNotFound): + // Other error occurred return err + case errors.Is(err, datastore.ErrNotFound): + // No sampling result found, select new samples + samples, err = SampleSquare(len(dah.RowRoots), int(la.params.SampleAmount)) + if err != nil { + return err + } + default: + // Sampling result found, unmarshal it + samples, err = decodeSamples(last) + if err != nil { + return err + } } - log.Debugw("validate availability", "root", dah.String()) - // We assume the caller of this method has already performed basic validation on the - // given dah/root. If for some reason this has not happened, the node should panic. if err := dah.ValidateBasic(); err != nil { - log.Errorw("availability validation cannot be performed on a malformed DataAvailabilityHeader", - "err", err) - panic(err) - } - samples, err := SampleSquare(len(dah.RowRoots), int(la.params.SampleAmount)) - if err != nil { + log.Errorw("DAH validation failed", "error", err) return err } @@ -94,49 +104,50 @@ func (la *ShareAvailability) SharesAvailable(ctx context.Context, header *header // functionality is optional and must be supported by the used share.Getter. ctx = getters.WithSession(ctx) + var ( + failedSamplesLock sync.Mutex + failedSamples []Sample + ) + log.Debugw("starting sampling session", "root", dah.String()) - errs := make(chan error, len(samples)) + var wg sync.WaitGroup for _, s := range samples { + wg.Add(1) go func(s Sample) { - log.Debugw("fetching share", "root", dah.String(), "row", s.Row, "col", s.Col) - _, err := la.getter.GetShare(ctx, header, s.Row, s.Col) + defer wg.Done() + // check if the sample is available + _, err := la.getter.GetShare(ctx, header, int(s.Row), int(s.Col)) if err != nil { log.Debugw("error fetching share", "root", dah.String(), "row", s.Row, "col", s.Col) - } - // we don't really care about Share bodies at this point - // it also means we now saved the Share in local storage - select { - case errs <- err: - case <-ctx.Done(): + failedSamplesLock.Lock() + failedSamples = append(failedSamples, s) + failedSamplesLock.Unlock() } }(s) } + wg.Wait() - for range samples { - var err error - select { - case err = <-errs: - case <-ctx.Done(): - err = ctx.Err() - } - - if err != nil { - if errors.Is(err, context.Canceled) { - return err - } - log.Errorw("availability validation failed", "root", dah.String(), "err", err.Error()) - if ipldFormat.IsNotFound(err) || errors.Is(err, context.DeadlineExceeded) { - return share.ErrNotAvailable - } - return err - } + if errors.Is(ctx.Err(), context.Canceled) { + // Availability did not complete due to context cancellation, return context error instead of share.ErrNotAvailable + return ctx.Err() } + // store the result of the sampling session + bs := encodeSamples(failedSamples) la.dsLk.Lock() - err = la.ds.Put(ctx, key, []byte{}) + err = la.ds.Put(ctx, key, bs) la.dsLk.Unlock() if err != nil { - log.Errorw("storing root of successful SharesAvailable request to disk", "err", err) + log.Errorw("Failed to store sampling result", "error", err) + } + + // if any of the samples failed, return an error + if len(failedSamples) > 0 { + log.Errorw("availability validation failed", + "root", dah.String(), + "failed_samples", failedSamples, + ) + return share.ErrNotAvailable } return nil } diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index 2ace654d50..68da3698b5 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -6,7 +6,6 @@ import ( "strconv" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-node/header/headertest" @@ -26,16 +25,18 @@ func TestSharesAvailableCaches(t *testing.T) { // cache doesn't have dah yet has, err := avail.ds.Has(ctx, rootKey(dah)) - assert.NoError(t, err) - assert.False(t, has) + require.NoError(t, err) + require.False(t, has) err = avail.SharesAvailable(ctx, eh) - assert.NoError(t, err) + require.NoError(t, err) - // is now cached - has, err = avail.ds.Has(ctx, rootKey(dah)) - assert.NoError(t, err) - assert.True(t, has) + // is now stored success result + result, err := avail.ds.Get(ctx, rootKey(dah)) + require.NoError(t, err) + failed, err := decodeSamples(result) + require.NoError(t, err) + require.Empty(t, failed) } func TestSharesAvailableHitsCache(t *testing.T) { @@ -45,19 +46,16 @@ func TestSharesAvailableHitsCache(t *testing.T) { getter, _ := GetterWithRandSquare(t, 16) avail := TestAvailability(getter) + // create new dah, that is not available by getter bServ := ipld.NewMemBlockservice() dah := availability_test.RandFillBS(t, 16, bServ) eh := headertest.RandExtendedHeaderWithRoot(t, dah) // blockstore doesn't actually have the dah err := avail.SharesAvailable(ctx, eh) - require.Error(t, err) - - // cache doesn't have dah yet, since it errored - has, err := avail.ds.Has(ctx, rootKey(dah)) - assert.NoError(t, err) - assert.False(t, has) + require.ErrorIs(t, err, share.ErrNotAvailable) + // put success result in cache err = avail.ds.Put(ctx, rootKey(dah), []byte{}) require.NoError(t, err) @@ -75,31 +73,47 @@ func TestSharesAvailableEmptyRoot(t *testing.T) { eh := headertest.RandExtendedHeaderWithRoot(t, share.EmptyRoot()) err := avail.SharesAvailable(ctx, eh) - assert.NoError(t, err) + require.NoError(t, err) } -func TestSharesAvailable(t *testing.T) { +func TestSharesAvailableFailed(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - getter, dah := GetterWithRandSquare(t, 16) + getter, _ := GetterWithRandSquare(t, 16) avail := TestAvailability(getter) - err := avail.SharesAvailable(ctx, dah) - assert.NoError(t, err) -} - -func TestSharesAvailableFailed(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + // create new dah, that is not available by getter bServ := ipld.NewMemBlockservice() dah := availability_test.RandFillBS(t, 16, bServ) eh := headertest.RandExtendedHeaderWithRoot(t, dah) - getter, _ := GetterWithRandSquare(t, 16) - avail := TestAvailability(getter) + // blockstore doesn't actually have the dah, so it should fail err := avail.SharesAvailable(ctx, eh) - assert.Error(t, err) + require.ErrorIs(t, err, share.ErrNotAvailable) + + // cache should have failed results now + result, err := avail.ds.Get(ctx, rootKey(dah)) + require.NoError(t, err) + + failed, err := decodeSamples(result) + require.NoError(t, err) + require.Len(t, failed, int(avail.params.SampleAmount)) + + // ensure that retry persists the failed samples selection + // create new getter with only the failed samples available, and add them to the onceGetter + onceGetter := newOnceGetter() + onceGetter.AddSamples(failed) + + // replace getter with the new one + avail.getter = onceGetter + + // should be able to retrieve all the failed samples now + err = avail.SharesAvailable(ctx, eh) + require.NoError(t, err) + + // onceGetter should have no more samples stored after the call + require.Empty(t, onceGetter.available) } func TestShareAvailableOverMocknet_Light(t *testing.T) { @@ -114,7 +128,7 @@ func TestShareAvailableOverMocknet_Light(t *testing.T) { net.ConnectAll() err := nd.SharesAvailable(ctx, eh) - assert.NoError(t, err) + require.NoError(t, err) } func TestGetShare(t *testing.T) { @@ -127,8 +141,8 @@ func TestGetShare(t *testing.T) { for i := range make([]bool, n) { for j := range make([]bool, n) { sh, err := getter.GetShare(ctx, eh, i, j) - assert.NotNil(t, sh) - assert.NoError(t, err) + require.NotNil(t, sh) + require.NoError(t, err) } } } @@ -163,14 +177,14 @@ func TestService_GetSharesByNamespace(t *testing.T) { require.NoError(t, err) require.NoError(t, shares.Verify(root, randNamespace)) flattened := shares.Flatten() - assert.Len(t, flattened, tt.expectedShareCount) + require.Len(t, flattened, tt.expectedShareCount) for _, value := range flattened { - assert.Equal(t, randNamespace, share.GetNamespace(value)) + require.Equal(t, randNamespace, share.GetNamespace(value)) } if tt.expectedShareCount > 1 { // idx1 is always smaller than idx2 - assert.Equal(t, randShares[idx1], flattened[0]) - assert.Equal(t, randShares[idx2], flattened[1]) + require.Equal(t, randShares[idx1], flattened[0]) + require.Equal(t, randShares[idx2], flattened[1]) } }) t.Run("last two rows of a 4x4 square that have the same namespace have valid NMT proofs", func(t *testing.T) { diff --git a/share/availability/light/sample.go b/share/availability/light/sample.go index e66ff9aafe..e09a46a5fc 100644 --- a/share/availability/light/sample.go +++ b/share/availability/light/sample.go @@ -3,12 +3,14 @@ package light import ( crand "crypto/rand" + "encoding/binary" + "errors" "math/big" ) // Sample is a point in 2D space over square. type Sample struct { - Row, Col int + Row, Col uint16 } // SampleSquare randomly picks *num* unique points from the given *width* square @@ -66,11 +68,37 @@ func (ss *squareSampler) samples() []Sample { return samples } -func randInt(max int) int { +func randInt(max int) uint16 { n, err := crand.Int(crand.Reader, big.NewInt(int64(max))) if err != nil { panic(err) // won't panic as rand.Reader is endless } - return int(n.Int64()) + return uint16(n.Uint64()) +} + +// encodeSamples encodes a slice of samples into a byte slice using little endian encoding. +func encodeSamples(samples []Sample) []byte { + bs := make([]byte, 0, len(samples)*4) + for _, s := range samples { + bs = binary.LittleEndian.AppendUint16(bs, s.Row) + bs = binary.LittleEndian.AppendUint16(bs, s.Col) + } + return bs +} + +// decodeSamples decodes a byte slice into a slice of samples. +func decodeSamples(bs []byte) ([]Sample, error) { + if len(bs)%4 != 0 { + return nil, errors.New("invalid byte slice length") + } + + samples := make([]Sample, 0, len(bs)/4) + for i := 0; i < len(bs); i += 4 { + samples = append(samples, Sample{ + Row: binary.LittleEndian.Uint16(bs[i : i+2]), + Col: binary.LittleEndian.Uint16(bs[i+2 : i+4]), + }) + } + return samples, nil } diff --git a/share/availability/light/sample_test.go b/share/availability/light/sample_test.go index 7092b99e83..8d7656e688 100644 --- a/share/availability/light/sample_test.go +++ b/share/availability/light/sample_test.go @@ -21,8 +21,8 @@ func TestSampleSquare(t *testing.T) { assert.Len(t, ss, tt.samples) // check points are within width for _, s := range ss { - assert.Less(t, s.Row, tt.width) - assert.Less(t, s.Col, tt.width) + assert.Less(t, int(s.Row), tt.width) + assert.Less(t, int(s.Col), tt.width) } // checks samples are not equal for i, s1 := range ss { diff --git a/share/availability/light/testing.go b/share/availability/light/testing.go index 9efc9ff14a..b6251b4fbd 100644 --- a/share/availability/light/testing.go +++ b/share/availability/light/testing.go @@ -1,11 +1,15 @@ package light import ( + "context" + "sync" "testing" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-datastore" + "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" @@ -58,3 +62,46 @@ func SubNetNode(sn *availability_test.SubNet) *availability_test.TestNode { sn.AddNode(nd) return nd } + +type onceGetter struct { + *sync.Mutex + available map[Sample]struct{} +} + +func newOnceGetter() onceGetter { + return onceGetter{ + Mutex: &sync.Mutex{}, + available: make(map[Sample]struct{}), + } +} + +func (m onceGetter) AddSamples(samples []Sample) { + m.Lock() + defer m.Unlock() + for _, s := range samples { + m.available[s] = struct{}{} + } +} + +func (m onceGetter) GetShare(_ context.Context, _ *header.ExtendedHeader, row, col int) (share.Share, error) { + m.Lock() + defer m.Unlock() + s := Sample{Row: uint16(row), Col: uint16(col)} + if _, ok := m.available[s]; ok { + delete(m.available, s) + return share.Share{}, nil + } + return share.Share{}, share.ErrNotAvailable +} + +func (m onceGetter) GetEDS(_ context.Context, _ *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { + panic("not implemented") +} + +func (m onceGetter) GetSharesByNamespace( + _ context.Context, + _ *header.ExtendedHeader, + _ share.Namespace, +) (share.NamespacedShares, error) { + panic("not implemented") +} diff --git a/share/eds/cache/metrics.go b/share/eds/cache/metrics.go index b2e3bec8d8..565a61a5e0 100644 --- a/share/eds/cache/metrics.go +++ b/share/eds/cache/metrics.go @@ -39,7 +39,7 @@ func newMetrics(bc *AccessorCache) (*metrics, error) { return nil, err } - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { observer.ObserveInt64(cacheSize, int64(bc.cache.Len())) return nil } diff --git a/share/eds/metrics.go b/share/eds/metrics.go index 8d69a3ec41..0fd6740154 100644 --- a/share/eds/metrics.go +++ b/share/eds/metrics.go @@ -128,7 +128,7 @@ func (s *Store) WithMetrics() error { return err } - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { stats := s.dgstr.Stats() for status, amount := range stats { observer.ObserveInt64(dagStoreShards, int64(amount), diff --git a/share/ipld/add.go b/share/ipld/add.go index fbb743148c..4dc5f1cf47 100644 --- a/share/ipld/add.go +++ b/share/ipld/add.go @@ -47,7 +47,7 @@ func AddShares( return eds, batchAdder.Commit() } -// ImportShares imports flattened chunks of data into Extended Data square and saves it in +// ImportShares imports flattened pieces of data into Extended Data square and saves it in // blockservice.BlockService func ImportShares( ctx context.Context, diff --git a/share/p2p/discovery/metrics.go b/share/p2p/discovery/metrics.go index 78b62a7d97..c91e68491d 100644 --- a/share/p2p/discovery/metrics.go +++ b/share/p2p/discovery/metrics.go @@ -102,7 +102,7 @@ func initMetrics(d *Discovery) (*metrics, error) { peerRemoved: peerRemoved, } - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { observer.ObserveInt64(peersAmount, int64(d.set.Size())) observer.ObserveInt64(backOffSize, int64(d.connector.Size())) return nil diff --git a/share/p2p/peers/metrics.go b/share/p2p/peers/metrics.go index eb42254430..094d81a5e3 100644 --- a/share/p2p/peers/metrics.go +++ b/share/p2p/peers/metrics.go @@ -130,7 +130,7 @@ func initMetrics(manager *Manager) (*metrics, error) { blacklistedPeers: blacklisted, } - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { for poolStatus, count := range manager.shrexPools() { observer.ObserveInt64(shrexPools, count, metric.WithAttributes( diff --git a/state/metrics.go b/state/metrics.go index aa166e901d..3672ef9b36 100644 --- a/state/metrics.go +++ b/state/metrics.go @@ -19,7 +19,7 @@ func WithMetrics(ca *CoreAccessor) { metric.WithDescription("Timestamp of the last submitted PayForBlob transaction"), ) - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { observer.ObserveInt64(pfbCounter, ca.PayForBlobCount()) observer.ObserveInt64(lastPfbTimestamp, ca.LastPayForBlob()) return nil