From 57e1cab93b839cb69fa96e2bc4503475abf439e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:21:10 +0000 Subject: [PATCH 01/25] chore(deps): Bump go.uber.org/zap from 1.26.0 to 1.27.0 (#3209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [//]: # (dependabot-start) ⚠️ **Dependabot is rebasing this PR** ⚠️ Rebasing might not happen immediately, so don't worry if this takes some time. Note: if you make any changes to this PR yourself, they will take precedence over the rebase. --- [//]: # (dependabot-end) Bumps [go.uber.org/zap](https://github.com/uber-go/zap) from 1.26.0 to 1.27.0.
Release notes

Sourced from go.uber.org/zap's releases.

v1.27.0

Enhancements:

Thanks to @​defval, @​dimmo, @​arxeiss, and @​MKrupauskas for their contributions to this release.

#1378: uber-go/zap#1378 #1399: uber-go/zap#1399 #1406: uber-go/zap#1406 #1416: uber-go/zap#1416

Changelog

Sourced from go.uber.org/zap's changelog.

1.27.0 (20 Feb 2024)

Enhancements:

Thanks to @​defval, @​dimmo, @​arxeiss, and @​MKrupauskas for their contributions to this release.

#1378: uber-go/zap#1378 #1399: uber-go/zap#1399 #1406: uber-go/zap#1406 #1416: uber-go/zap#1416

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=go.uber.org/zap&package-manager=go_modules&previous-version=1.26.0&new-version=1.27.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index da96f82757..727c73fabc 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 go.opentelemetry.io/proto/otlp v1.1.0 go.uber.org/fx v1.20.1 - go.uber.org/zap v1.26.0 + go.uber.org/zap v1.27.0 golang.org/x/crypto v0.19.0 golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e golang.org/x/sync v0.6.0 diff --git a/go.sum b/go.sum index a5ea60f868..75c6dd158a 100644 --- a/go.sum +++ b/go.sum @@ -2449,8 +2449,8 @@ go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -2474,8 +2474,8 @@ go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= From f2b664f3abe5e76ed51e19305aa3489980a93692 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Fri, 1 Mar 2024 12:51:49 +0200 Subject: [PATCH 02/25] feat(blob): extend blob struct with index field (#3165) What was done: * added index to the blob struct; * introduced a new `parser` entity that collects shares and transforms them to blob; * added tests to compare shares received from share service by coordinates that have gotten from blob index; --- blob/blob.go | 59 ++++----- blob/blob_test.go | 15 ++- blob/helper.go | 70 +++-------- blob/parser.go | 151 +++++++++++++++++++++++ blob/service.go | 219 +++++++++++++++++++++------------ blob/service_test.go | 104 ++++++++++++++-- nodebuilder/blob/cmd/blob.go | 3 + nodebuilder/tests/blob_test.go | 6 +- 8 files changed, 441 insertions(+), 186 deletions(-) create mode 100644 blob/parser.go diff --git a/blob/blob.go b/blob/blob.go index 9843441dd2..dc18669e32 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -9,13 +9,14 @@ import ( tmproto "github.com/tendermint/tendermint/proto/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/nmt" "github.com/celestiaorg/celestia-node/share" ) +var errEmptyShares = errors.New("empty shares") + // Commitment is a Merkle Root of the subtree built from shares of the Blob. // It is computed by splitting the blob into shares and building the Merkle subtree to be included // after Submit. @@ -31,6 +32,9 @@ func (com Commitment) Equal(c Commitment) bool { } // Proof is a collection of nmt.Proofs that verifies the inclusion of the data. +// Proof proves the WHOLE namespaced data for the particular row. +// TODO (@vgonkivs): rework `Proof` in order to prove a particular blob. +// https://github.com/celestiaorg/celestia-node/issues/2303 type Proof []*nmt.Proof func (p Proof) Len() int { return len(p) } @@ -99,6 +103,10 @@ type Blob struct { // the celestia-node's namespace type // this is to avoid converting to and from app's type namespace share.Namespace + + // index represents the index of the blob's first share in the EDS. + // Only retrieved, on-chain blobs will have the index set. Default is -1. + index int } // NewBlobV0 constructs a new blob from the provided Namespace and data. @@ -127,7 +135,7 @@ func NewBlob(shareVersion uint8, namespace share.Namespace, data []byte) (*Blob, if err != nil { return nil, err } - return &Blob{Blob: blob, Commitment: com, namespace: namespace}, nil + return &Blob{Blob: blob, Commitment: com, namespace: namespace, index: -1}, nil } // Namespace returns blob's namespace. @@ -135,11 +143,22 @@ func (b *Blob) Namespace() share.Namespace { return b.namespace } +// Index returns the blob's first share index in the EDS. +// Only retrieved, on-chain blobs will have the index set. Default is -1. +func (b *Blob) Index() int { + return b.index +} + +func (b *Blob) compareCommitments(com Commitment) bool { + return bytes.Equal(b.Commitment, com) +} + type jsonBlob struct { Namespace share.Namespace `json:"namespace"` Data []byte `json:"data"` ShareVersion uint32 `json:"share_version"` Commitment Commitment `json:"commitment"` + Index int `json:"index"` } func (b *Blob) MarshalJSON() ([]byte, error) { @@ -148,6 +167,7 @@ func (b *Blob) MarshalJSON() ([]byte, error) { Data: b.Data, ShareVersion: b.ShareVersion, Commitment: b.Commitment, + Index: b.index, } return json.Marshal(blob) } @@ -165,39 +185,6 @@ func (b *Blob) UnmarshalJSON(data []byte) error { b.Blob.ShareVersion = blob.ShareVersion b.Commitment = blob.Commitment b.namespace = blob.Namespace + b.index = blob.Index return nil } - -// buildBlobsIfExist takes shares and tries building the Blobs from them. -// It will build blobs either until appShares will be empty or the first incomplete blob will -// appear, so in this specific case it will return all built blobs + remaining shares. -func buildBlobsIfExist(appShares []shares.Share) ([]*Blob, []shares.Share, error) { - if len(appShares) == 0 { - return nil, nil, errors.New("empty shares received") - } - blobs := make([]*Blob, 0, len(appShares)) - for { - length, err := appShares[0].SequenceLen() - if err != nil { - return nil, nil, err - } - - amount := shares.SparseSharesNeeded(length) - if amount > len(appShares) { - return blobs, appShares, nil - } - - b, err := parseShares(appShares[:amount]) - if err != nil { - return nil, nil, err - } - - // only 1 blob will be created bc we passed the exact amount of shares - blobs = append(blobs, b[0]) - - if amount == len(appShares) { - return blobs, nil, nil - } - appShares = appShares[amount:] - } -} diff --git a/blob/blob_test.go b/blob/blob_test.go index 85486ad125..020bfb191b 100644 --- a/blob/blob_test.go +++ b/blob/blob_test.go @@ -1,7 +1,7 @@ package blob import ( - "reflect" + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -14,7 +14,7 @@ import ( ) func TestBlob(t *testing.T) { - appBlobs, err := blobtest.GenerateV0Blobs([]int{1}, false) + appBlobs, err := blobtest.GenerateV0Blobs([]int{16}, false) require.NoError(t, err) blob, err := convertBlobs(appBlobs...) require.NoError(t, err) @@ -53,10 +53,12 @@ func TestBlob(t *testing.T) { expectedRes: func(t *testing.T) { sh, err := BlobsToShares(blob...) require.NoError(t, err) - b, err := SharesToBlobs(sh) + shares, err := toAppShares(sh...) require.NoError(t, err) - assert.Equal(t, len(b), 1) - assert.Equal(t, blob[0].Commitment, b[0].Commitment) + p := &parser{length: len(shares), shares: shares} + b, err := p.parse() + require.NoError(t, err) + assert.Equal(t, blob[0].Commitment, b.Commitment) }, }, { @@ -67,7 +69,8 @@ func TestBlob(t *testing.T) { newBlob := &Blob{} require.NoError(t, newBlob.UnmarshalJSON(data)) - require.True(t, reflect.DeepEqual(blob[0], newBlob)) + require.True(t, bytes.Equal(blob[0].Blob.Data, newBlob.Data)) + require.True(t, bytes.Equal(blob[0].Commitment, newBlob.Commitment)) }, }, } diff --git a/blob/helper.go b/blob/helper.go index 72a56c7889..d3b418e8ba 100644 --- a/blob/helper.go +++ b/blob/helper.go @@ -11,55 +11,6 @@ import ( "github.com/celestiaorg/celestia-node/share" ) -// SharesToBlobs takes raw shares and converts them to the blobs. -func SharesToBlobs(rawShares []share.Share) ([]*Blob, error) { - if len(rawShares) == 0 { - return nil, ErrBlobNotFound - } - - appShares, err := toAppShares(rawShares...) - if err != nil { - return nil, err - } - return parseShares(appShares) -} - -// parseShares takes shares and converts them to the []*Blob. -func parseShares(appShrs []shares.Share) ([]*Blob, error) { - shareSequences, err := shares.ParseShares(appShrs, true) - if err != nil { - return nil, err - } - - // ensure that sequence length is not 0 - if len(shareSequences) == 0 { - return nil, ErrBlobNotFound - } - - blobs := make([]*Blob, len(shareSequences)) - for i, sequence := range shareSequences { - data, err := sequence.RawData() - if err != nil { - return nil, err - } - if len(data) == 0 { - continue - } - - shareVersion, err := sequence.Shares[0].Version() - if err != nil { - return nil, err - } - - blob, err := NewBlob(shareVersion, sequence.Namespace.Bytes(), data) - if err != nil { - return nil, err - } - blobs[i] = blob - } - return blobs, nil -} - // BlobsToShares accepts blobs and convert them to the Shares. func BlobsToShares(blobs ...*Blob) ([]share.Share, error) { b := make([]types.Blob, len(blobs)) @@ -75,7 +26,7 @@ func BlobsToShares(blobs ...*Blob) ([]share.Share, error) { sort.Slice(b, func(i, j int) bool { val := bytes.Compare(b[i].NamespaceID, b[j].NamespaceID) - return val <= 0 + return val < 0 }) rawShares, err := shares.SplitBlobs(b...) @@ -84,3 +35,22 @@ func BlobsToShares(blobs ...*Blob) ([]share.Share, error) { } return shares.ToBytes(rawShares), nil } + +// toAppShares converts node's raw shares to the app shares, skipping padding +func toAppShares(shrs ...share.Share) ([]shares.Share, error) { + appShrs := make([]shares.Share, 0, len(shrs)) + for _, shr := range shrs { + bShare, err := shares.NewShare(shr) + if err != nil { + return nil, err + } + appShrs = append(appShrs, *bShare) + } + return appShrs, nil +} + +func calculateIndex(rowLength, blobIndex int) (row, col int) { + row = blobIndex / rowLength + col = blobIndex - (row * rowLength) + return +} diff --git a/blob/parser.go b/blob/parser.go new file mode 100644 index 0000000000..8999afcb22 --- /dev/null +++ b/blob/parser.go @@ -0,0 +1,151 @@ +package blob + +import ( + "errors" + "fmt" + + "github.com/celestiaorg/celestia-app/pkg/shares" +) + +// parser is a helper struct that allows collecting shares and transforming them into the blob. +// it contains all necessary information that is needed to build the blob: +// * position of the blob inside the EDS; +// * blob's length; +// * shares needed to build the blob; +// * extra condition to verify the final blob. +type parser struct { + index int + length int + shares []shares.Share + verifyFn func(blob *Blob) bool +} + +// NOTE: passing shares here needed to detect padding shares(as we do not need this check in addShares) +func (p *parser) set(index int, shrs []shares.Share) ([]shares.Share, error) { + if len(shrs) == 0 { + return nil, errEmptyShares + } + + shrs, err := p.skipPadding(shrs) + if err != nil { + return nil, err + } + + if len(shrs) == 0 { + return nil, errEmptyShares + } + + // `+=` as index could be updated in `skipPadding` + p.index += index + length, err := shrs[0].SequenceLen() + if err != nil { + return nil, err + } + + p.length = shares.SparseSharesNeeded(length) + return shrs, nil +} + +// addShares sets shares until the blob is completed and returns extra shares back. +// we do not need here extra condition to check padding shares as we do not expect it here. +// it is possible only between two blobs. +func (p *parser) addShares(shares []shares.Share) (shrs []shares.Share, isComplete bool) { + index := -1 + for i, sh := range shares { + p.shares = append(p.shares, sh) + if len(p.shares) == p.length { + index = i + isComplete = true + break + } + } + + if index == -1 { + return + } + + if index+1 >= len(shares) { + return shrs, true + } + return shares[index+1:], true +} + +// parse parses shares and creates the Blob. +func (p *parser) parse() (*Blob, error) { + if p.length != len(p.shares) { + return nil, fmt.Errorf("invalid shares amount. want:%d, have:%d", p.length, len(p.shares)) + } + + sequence, err := shares.ParseShares(p.shares, true) + if err != nil { + return nil, err + } + + // ensure that sequence length is not 0 + if len(sequence) == 0 { + return nil, ErrBlobNotFound + } + if len(sequence) > 1 { + return nil, errors.New("unexpected amount of sequences") + } + + data, err := sequence[0].RawData() + if err != nil { + return nil, err + } + if len(data) == 0 { + return nil, ErrBlobNotFound + } + + shareVersion, err := sequence[0].Shares[0].Version() + if err != nil { + return nil, err + } + + blob, err := NewBlob(shareVersion, sequence[0].Namespace.Bytes(), data) + if err != nil { + return nil, err + } + blob.index = p.index + return blob, nil +} + +// skipPadding skips first share in the range if this share is the Padding share. +func (p *parser) skipPadding(shares []shares.Share) ([]shares.Share, error) { + if len(shares) == 0 { + return nil, errEmptyShares + } + + isPadding, err := shares[0].IsPadding() + if err != nil { + return nil, err + } + + if !isPadding { + return shares, nil + } + + // update blob index if we are going to skip one share + p.index++ + if len(shares) > 1 { + return shares[1:], nil + } + return nil, nil +} + +func (p *parser) verify(blob *Blob) bool { + if p.verifyFn == nil { + return false + } + return p.verifyFn(blob) +} + +func (p *parser) isEmpty() bool { + return p.index == 0 && p.length == 0 && len(p.shares) == 0 +} + +func (p *parser) reset() { + p.index = 0 + p.length = 0 + p.shares = nil +} diff --git a/blob/service.go b/blob/service.go index fc1d630e62..7445c05054 100644 --- a/blob/service.go +++ b/blob/service.go @@ -111,12 +111,27 @@ func (s *Service) Submit(ctx context.Context, blobs []*Blob, gasPrice GasPrice) } // Get retrieves all the blobs for given namespaces at the given height by commitment. -func (s *Service) Get(ctx context.Context, height uint64, ns share.Namespace, commitment Commitment) (*Blob, error) { - blob, _, err := s.getByCommitment(ctx, height, ns, commitment) - if err != nil { - return nil, err - } - return blob, nil +func (s *Service) Get( + ctx context.Context, + height uint64, + namespace share.Namespace, + commitment Commitment, +) (blob *Blob, err error) { + ctx, span := tracer.Start(ctx, "get") + defer func() { + utils.SetStatusAndEnd(span, err) + }() + span.SetAttributes( + attribute.Int64("height", int64(height)), + attribute.String("namespace", namespace.String()), + ) + + sharesParser := &parser{verifyFn: func(blob *Blob) bool { + return blob.compareCommitments(commitment) + }} + + blob, _, err = s.retrieve(ctx, height, namespace, sharesParser) + return } // GetProof retrieves all blobs in the given namespaces at the given height by commitment @@ -126,11 +141,21 @@ func (s *Service) GetProof( height uint64, namespace share.Namespace, commitment Commitment, -) (*Proof, error) { - _, proof, err := s.getByCommitment(ctx, height, namespace, commitment) - if err != nil { - return nil, err - } +) (proof *Proof, err error) { + ctx, span := tracer.Start(ctx, "get-proof") + defer func() { + utils.SetStatusAndEnd(span, err) + }() + span.SetAttributes( + attribute.Int64("height", int64(height)), + attribute.String("namespace", namespace.String()), + ) + + sharesParser := &parser{verifyFn: func(blob *Blob) bool { + return blob.compareCommitments(commitment) + }} + + _, proof, err = s.retrieve(ctx, height, namespace, sharesParser) return proof, nil } @@ -147,8 +172,8 @@ func (s *Service) GetAll(ctx context.Context, height uint64, namespaces []share. resultErr = make([]error, len(namespaces)) ) - for _, ns := range namespaces { - log.Debugw("performing GetAll request", "namespace", ns.String(), "height", height) + for _, namespace := range namespaces { + log.Debugw("performing GetAll request", "namespace", namespace.String(), "height", height) } wg := sync.WaitGroup{} @@ -190,12 +215,17 @@ func (s *Service) Included( height uint64, namespace share.Namespace, proof *Proof, - com Commitment, + commitment Commitment, ) (_ bool, err error) { ctx, span := tracer.Start(ctx, "included") defer func() { utils.SetStatusAndEnd(span, err) }() + span.SetAttributes( + attribute.Int64("height", int64(height)), + attribute.String("namespace", namespace.String()), + ) + // In the current implementation, LNs will have to download all shares to recompute the commitment. // To achieve 1. we need to modify Proof structure and to store all subtree roots, that were // involved in commitment creation and then call `merkle.HashFromByteSlices`(tendermint package). @@ -205,7 +235,10 @@ func (s *Service) Included( // but we have to guarantee that all our stored subtree roots will be on the same height(e.g. one // level above shares). // TODO(@vgonkivs): rework the implementation to perform all verification without network requests. - _, resProof, err := s.getByCommitment(ctx, height, namespace, com) + sharesParser := &parser{verifyFn: func(blob *Blob) bool { + return blob.compareCommitments(commitment) + }} + _, resProof, err := s.retrieve(ctx, height, namespace, sharesParser) switch err { case nil: case ErrBlobNotFound: @@ -216,27 +249,19 @@ func (s *Service) Included( return true, resProof.equal(*proof) } -// getByCommitment retrieves the DAH row by row, fetching shares and constructing blobs in order to -// compare Commitments. Retrieving is stopped once the requested blob/proof is found. -func (s *Service) getByCommitment( +// retrieve retrieves blobs and their proofs by requesting the whole namespace and +// comparing Commitments with each blob. +// Retrieving is stopped once the requested blob/proof is found. +func (s *Service) retrieve( ctx context.Context, height uint64, namespace share.Namespace, - commitment Commitment, + sharesParser *parser, ) (_ *Blob, _ *Proof, err error) { log.Infow("requesting blob", "height", height, "namespace", namespace.String()) - ctx, span := tracer.Start(ctx, "get-by-commitment") - defer func() { - utils.SetStatusAndEnd(span, err) - }() - span.SetAttributes( - attribute.Int64("height", int64(height)), - attribute.String("commitment", string(commitment)), - ) - getCtx, headerGetterSpan := tracer.Start(ctx, "header-getter") header, err := s.headerGetter(getCtx, height) @@ -249,6 +274,14 @@ func (s *Service) getByCommitment( headerGetterSpan.AddEvent("received eds", trace.WithAttributes( attribute.Int64("eds-size", int64(len(header.DAH.RowRoots))))) + rowIndex := -1 + for i, row := range header.DAH.RowRoots { + if !namespace.IsOutsideRange(row, row) { + rowIndex = i + break + } + } + getCtx, getSharesSpan := tracer.Start(ctx, "get-shares-by-namespace") namespacedShares, err := s.shareGetter.GetSharesByNamespace(getCtx, header, namespace) @@ -265,10 +298,8 @@ func (s *Service) getByCommitment( attribute.Int64("eds-size", int64(len(header.DAH.RowRoots))))) var ( - rawShares = make([]shares.Share, 0) + appShares = make([]shares.Share, 0) proofs = make(Proof, 0) - // spansMultipleRows specifies whether blob is expanded into multiple rows - spansMultipleRows bool ) for _, row := range namespacedShares { @@ -279,47 +310,81 @@ func (s *Service) getByCommitment( return nil, nil, ErrBlobNotFound } - appShares, err := toAppShares(row.Shares...) + appShares, err = toAppShares(row.Shares...) if err != nil { return nil, nil, err } - rawShares = append(rawShares, appShares...) + proofs = append(proofs, row.Proof) + index := row.Proof.Start() + + for { + var ( + isComplete bool + shrs []shares.Share + wasEmpty = sharesParser.isEmpty() + ) + + if wasEmpty { + // create a parser if it is empty + shrs, err = sharesParser.set(rowIndex*len(header.DAH.RowRoots)+index, appShares) + if err != nil { + if errors.Is(err, errEmptyShares) { + appShares = nil + break + } + return nil, nil, err + } + + if len(appShares) != len(shrs) { + // update index and shares if a padding share was detected. + index += len(appShares) - len(shrs) + appShares = shrs + } + } - var blobs []*Blob - blobs, rawShares, err = buildBlobsIfExist(rawShares) - if err != nil { - return nil, nil, err - } - for _, b := range blobs { - if b.Commitment.Equal(commitment) { - span.AddEvent("blob reconstructed") - return b, &proofs, nil + shrs, isComplete = sharesParser.addShares(appShares) + if !isComplete { + appShares = nil + break } - // Falling under this flag means that the data from the last row - // was insufficient to create a complete blob. As a result, - // the first blob received spans two rows and includes proofs - // for both of these rows. All other blobs in the result will relate - // to the current row and have a single proof. - if spansMultipleRows { - spansMultipleRows = false - // leave proof only for the current row + + blob, err := sharesParser.parse() + if err != nil { + return nil, nil, err + } + + if sharesParser.verify(blob) { + return blob, &proofs, nil + } + + index += len(appShares) - len(shrs) + appShares = shrs + sharesParser.reset() + + if !wasEmpty { + // remove proofs for prev rows if verified blob spans multiple rows proofs = proofs[len(proofs)-1:] } } - if len(rawShares) > 0 { - spansMultipleRows = true - continue + rowIndex++ + if sharesParser.isEmpty() { + proofs = nil } - proofs = nil } err = ErrBlobNotFound - if len(rawShares) > 0 { - err = fmt.Errorf("incomplete blob with the "+ - "namespace: %s detected at %d: %w", namespace.String(), height, err) - log.Error(err) + for _, sh := range appShares { + ok, err := sh.IsPadding() + if err != nil { + return nil, nil, err + } + if !ok { + err = fmt.Errorf("incomplete blob with the "+ + "namespace: %s detected at %d: %w", namespace.String(), height, err) + log.Error(err) + } } return nil, nil, err } @@ -332,34 +397,24 @@ func (s *Service) getBlobs( header *header.ExtendedHeader, ) (_ []*Blob, err error) { ctx, span := tracer.Start(ctx, "get-blobs") + span.SetAttributes( + attribute.Int64("height", int64(header.Height())), + attribute.String("namespace", namespace.String()), + ) defer func() { utils.SetStatusAndEnd(span, err) }() - namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header, namespace) - if err != nil { - return nil, err - } - return SharesToBlobs(namespacedShares.Flatten()) -} -// toAppShares converts node's raw shares to the app shares, skipping padding -func toAppShares(shrs ...share.Share) ([]shares.Share, error) { - appShrs := make([]shares.Share, 0, len(shrs)) - for _, shr := range shrs { - bShare, err := shares.NewShare(shr) - if err != nil { - return nil, err - } - - ok, err := bShare.IsPadding() - if err != nil { - return nil, err - } - if ok { - continue - } + blobs := make([]*Blob, 0) + verifyFn := func(blob *Blob) bool { + blobs = append(blobs, blob) + return false + } + sharesParser := &parser{verifyFn: verifyFn} - appShrs = append(appShrs, *bShare) + _, _, err = s.retrieve(ctx, header.Height(), namespace, sharesParser) + if len(blobs) == 0 { + return nil, ErrBlobNotFound } - return appShrs, nil + return blobs, nil } diff --git a/blob/service_test.go b/blob/service_test.go index 3e22f887af..4bda8d993b 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -5,6 +5,8 @@ import ( "context" "crypto/sha256" "encoding/json" + "fmt" + "sort" "testing" "time" @@ -72,8 +74,7 @@ func TestBlobService_Get(t *testing.T) { { name: "get all with the same namespace", doFn: func() (interface{}, error) { - b, err := service.GetAll(ctx, 1, []share.Namespace{blobs1[0].Namespace()}) - return b, err + return service.GetAll(ctx, 1, []share.Namespace{blobs1[0].Namespace()}) }, expectedResult: func(res interface{}, err error) { require.NoError(t, err) @@ -85,7 +86,47 @@ func TestBlobService_Get(t *testing.T) { assert.Len(t, blobs, 2) for i := range blobs1 { - bytes.Equal(blobs1[i].Commitment, blobs[i].Commitment) + require.Equal(t, blobs1[i].Commitment, blobs[i].Commitment) + } + }, + }, + { + name: "verify indexes", + doFn: func() (interface{}, error) { + b0, err := service.Get(ctx, 1, blobs0[0].Namespace(), blobs0[0].Commitment) + require.NoError(t, err) + b1, err := service.Get(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment) + require.NoError(t, err) + b23, err := service.GetAll(ctx, 1, []share.Namespace{blobs1[0].Namespace()}) + require.NoError(t, err) + return []*Blob{b0, b1, b23[0], b23[1]}, nil + }, + expectedResult: func(res interface{}, err error) { + require.NoError(t, err) + blobs, ok := res.([]*Blob) + assert.True(t, ok) + assert.NotEmpty(t, blobs) + assert.Len(t, blobs, 4) + + sort.Slice(blobs, func(i, j int) bool { + val := bytes.Compare(blobs[i].NamespaceId, blobs[j].NamespaceId) + return val < 0 + }) + + h, err := service.headerGetter(ctx, 1) + require.NoError(t, err) + + resultShares, err := BlobsToShares(blobs...) + require.NoError(t, err) + shareOffset := 0 + for i := range blobs { + row, col := calculateIndex(len(h.DAH.RowRoots), blobs[i].index) + sh, err := service.shareGetter.GetShare(ctx, h, row, col) + require.NoError(t, err) + require.True(t, bytes.Equal(sh, resultShares[shareOffset]), + fmt.Sprintf("issue on %d attempt. ROW:%d, COL: %d, blobIndex:%d", i, row, col, blobs[i].index), + ) + shareOffset += shares.SparseSharesNeeded(uint32(len(blobs[i].Data))) } }, }, @@ -300,7 +341,7 @@ func TestBlobService_Get(t *testing.T) { // But to satisfy the rule of eds creating, padding namespace share is placed between // blobs. Test ensures that blob service will skip padding share and return the correct blob. func TestService_GetSingleBlobWithoutPadding(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) t.Cleanup(cancel) appBlob, err := blobtest.GenerateV0Blobs([]int{9, 5}, true) @@ -342,6 +383,14 @@ func TestService_GetSingleBlobWithoutPadding(t *testing.T) { newBlob, err := service.Get(ctx, 1, blobs[1].Namespace(), blobs[1].Commitment) require.NoError(t, err) assert.Equal(t, newBlob.Commitment, blobs[1].Commitment) + + resultShares, err := BlobsToShares(newBlob) + require.NoError(t, err) + row, col := calculateIndex(len(h.DAH.RowRoots), newBlob.index) + sh, err := service.shareGetter.GetShare(ctx, h, row, col) + require.NoError(t, err) + + assert.Equal(t, sh, resultShares[0]) } func TestService_Get(t *testing.T) { @@ -356,10 +405,25 @@ func TestService_Get(t *testing.T) { require.NoError(t, err) service := createService(ctx, t, blobs) - for _, blob := range blobs { + + h, err := service.headerGetter(ctx, 1) + require.NoError(t, err) + + resultShares, err := BlobsToShares(blobs...) + require.NoError(t, err) + shareOffset := 0 + + for i, blob := range blobs { b, err := service.Get(ctx, 1, blob.Namespace(), blob.Commitment) require.NoError(t, err) assert.Equal(t, b.Commitment, blob.Commitment) + + row, col := calculateIndex(len(h.DAH.RowRoots), b.index) + sh, err := service.shareGetter.GetShare(ctx, h, row, col) + require.NoError(t, err) + + assert.Equal(t, sh, resultShares[shareOffset], fmt.Sprintf("issue on %d attempt", i)) + shareOffset += shares.SparseSharesNeeded(uint32(len(blob.Data))) } } @@ -410,11 +474,27 @@ func TestService_GetAllWithoutPadding(t *testing.T) { service := NewService(nil, getters.NewIPLDGetter(bs), fn) - _, err = service.GetAll(ctx, 1, []share.Namespace{blobs[0].Namespace(), blobs[1].Namespace()}) + blobs, err = service.GetAll(ctx, 1, []share.Namespace{blobs[0].Namespace(), blobs[1].Namespace()}) + require.NoError(t, err) + + resultShares, err := BlobsToShares(blobs...) require.NoError(t, err) + sort.Slice(blobs, func(i, j int) bool { + val := bytes.Compare(blobs[i].NamespaceId, blobs[j].NamespaceId) + return val < 0 + }) + shareOffset := 0 + for _, blob := range blobs { + row, col := calculateIndex(len(h.DAH.RowRoots), blob.index) + sh, err := service.shareGetter.GetShare(ctx, h, row, col) + require.NoError(t, err) + + assert.Equal(t, sh, resultShares[shareOffset]) + shareOffset += shares.SparseSharesNeeded(uint32(len(blob.Data))) + } } -// BenchmarkGetByCommitment-12 3139 380827 ns/op 701647 B/op 4990 allocs/op +// BenchmarkGetByCommitment-12 1869 571663 ns/op 1085371 B/op 6414 allocs/op func BenchmarkGetByCommitment(b *testing.B) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) b.Cleanup(cancel) @@ -425,11 +505,17 @@ func BenchmarkGetByCommitment(b *testing.B) { require.NoError(b, err) service := createService(ctx, b, blobs) + indexer := &parser{} b.ResetTimer() for i := 0; i < b.N; i++ { b.ReportAllocs() - _, _, err = service.getByCommitment( - ctx, 1, blobs[1].Namespace(), blobs[1].Commitment, + indexer.reset() + indexer.verifyFn = func(blob *Blob) bool { + return blob.compareCommitments(blobs[1].Commitment) + } + + _, _, err = service.retrieve( + ctx, 1, blobs[1].Namespace(), indexer, ) require.NoError(b, err) } diff --git a/nodebuilder/blob/cmd/blob.go b/nodebuilder/blob/cmd/blob.go index 25a102843b..78dd57f250 100644 --- a/nodebuilder/blob/cmd/blob.go +++ b/nodebuilder/blob/cmd/blob.go @@ -272,6 +272,7 @@ func formatData(data interface{}) interface{} { Data string `json:"data"` ShareVersion uint32 `json:"share_version"` Commitment []byte `json:"commitment"` + Index int `json:"index"` } if reflect.TypeOf(data).Kind() == reflect.Slice { @@ -283,6 +284,7 @@ func formatData(data interface{}) interface{} { Data: string(b.Data), ShareVersion: b.ShareVersion, Commitment: b.Commitment, + Index: b.Index(), } } return result @@ -294,5 +296,6 @@ func formatData(data interface{}) interface{} { Data: string(b.Data), ShareVersion: b.ShareVersion, Commitment: b.Commitment, + Index: b.Index(), } } diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index d0aeefd568..7eb225a14a 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -79,7 +79,7 @@ func TestBlobModule(t *testing.T) { time.Sleep(time.Second) blob1, err := fullClient.Blob.Get(ctx, height, blobs[0].Namespace(), blobs[0].Commitment) require.NoError(t, err) - require.Equal(t, blobs[0], blob1) + require.Equal(t, blobs[0].Commitment, blob1.Commitment) }, }, { @@ -151,7 +151,7 @@ func TestBlobModule(t *testing.T) { b0, err := fullClient.Blob.Get(ctx, height, b.Namespace(), b.Commitment) require.NoError(t, err) - require.Equal(t, b, b0) + require.Equal(t, b.Commitment, b0.Commitment) // give some time to store the data, // otherwise the test will hang on the IPLD level. @@ -180,7 +180,7 @@ func TestBlobModule(t *testing.T) { b0, err := fullClient.Blob.Get(ctx, h, blobs[0].Namespace(), blobs[0].Commitment) require.NoError(t, err) - require.Equal(t, blobs[0], b0) + require.Equal(t, blobs[0].Commitment, b0.Commitment) // give some time to store the data, // otherwise the test will hang on the IPLD level. From 2569c429a2e17647c9aa3c278851ebdaa01341f0 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Fri, 1 Mar 2024 13:05:59 -0500 Subject: [PATCH 03/25] chore(deps): upgrade to app v1.7.0-rc0 --- go.mod | 33 +++++++++++++------------- go.sum | 75 ++++++++++++++++++++++++++++++++++------------------------ 2 files changed, 61 insertions(+), 47 deletions(-) diff --git a/go.mod b/go.mod index 727c73fabc..a804c92778 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/celestiaorg/celestia-node -go 1.21.1 +go 1.22.0 require ( cosmossdk.io/errors v1.0.1 @@ -8,13 +8,13 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.4.0 + github.com/celestiaorg/celestia-app v1.7.0-rc0 github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.5.3 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.20.0 github.com/celestiaorg/rsmt2d v0.11.0 - github.com/cosmos/cosmos-sdk v0.46.14 + github.com/cosmos/cosmos-sdk v0.46.16 github.com/cosmos/cosmos-sdk/api v0.1.0 github.com/cristalhq/jwt v1.2.0 github.com/dgraph-io/badger/v4 v4.2.1-0.20240106094458-1c417aa3799c @@ -56,7 +56,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 - github.com/tendermint/tendermint v0.34.28 + github.com/tendermint/tendermint v0.34.29 go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.24.0 @@ -137,10 +137,10 @@ require ( github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/dvsekhvalnov/jose2go v1.5.0 // indirect + github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect - github.com/ethereum/c-kzg-4844 v0.3.1 // indirect + github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/ethereum/go-ethereum v1.13.2 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flynn/noise v1.0.1 // indirect @@ -149,7 +149,7 @@ require ( github.com/gammazero/deque v0.2.0 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -178,6 +178,9 @@ require ( github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/grafana/otel-profiling-go v0.5.1 // indirect + github.com/grafana/pyroscope-go v1.1.1 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect @@ -239,7 +242,7 @@ require ( github.com/libp2p/go-netroute v0.2.1 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.1 // indirect - github.com/magiconair/properties v1.8.6 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/manifoldco/promptui v0.9.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect @@ -267,7 +270,6 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect @@ -286,17 +288,16 @@ require ( github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/regen-network/cosmos-proto v0.3.1 // indirect - github.com/rivo/uniseg v0.4.4 // indirect - github.com/rs/cors v1.8.2 // indirect + github.com/rs/cors v1.8.3 // indirect github.com/rs/zerolog v1.31.0 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/shirou/gopsutil v3.21.6+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.9.2 // indirect + github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/viper v1.14.0 // indirect - github.com/subosito/gotenv v1.4.1 // indirect + github.com/spf13/viper v1.15.0 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/supranational/blst v0.3.11 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect @@ -344,10 +345,10 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.20.1-sdk-v0.46.16 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.35.0-tm-v0.34.29 ) diff --git a/go.sum b/go.sum index 75c6dd158a..a5e321b654 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOv github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -335,8 +335,8 @@ github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJ github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= -github.com/btcsuite/btcd/btcutil v1.1.2 h1:XLMbX8JQEiwMcYft2EGi8zPUkoa0abKIU6/BJSRsjzQ= -github.com/btcsuite/btcd/btcutil v1.1.2/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= +github.com/btcsuite/btcd/btcutil v1.1.3 h1:xfbtw8lwpp0G6NwSHb+UE67ryTFHJAiNuipusjXSohQ= +github.com/btcsuite/btcd/btcutil v1.1.3/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= @@ -352,18 +352,18 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/bufbuild/protocompile v0.1.0 h1:HjgJBI85hY/qmW5tw/66sNDZ7z0UDdVSi/5r40WHw4s= -github.com/bufbuild/protocompile v0.1.0/go.mod h1:ix/MMMdsT3fzxfw91dvbfzKW3fRRnuPCP47kpAm5m/4= +github.com/bufbuild/protocompile v0.5.1 h1:mixz5lJX4Hiz4FpqFREJHIXLfaLBntfaJv1h+/jS+Qg= +github.com/bufbuild/protocompile v0.5.1/go.mod h1:G5iLmavmF4NsYtpZFvE3B/zFch2GIY8+wjsYLR/lc40= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.4.0 h1:hTId3xL8GssN5sHSHPP7svHi/iWp+XVxphStiR7ADiY= -github.com/celestiaorg/celestia-app v1.4.0/go.mod h1:zhdQIFGFZRRxrDVtFE4OFIT7/12RE8DRyfvNZdW8ceM= -github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 h1:Fd7ymPUzExPGNl2gZw4i5S74arMw+iDHLE78M/cCxl4= -github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29/go.mod h1:xrICN0PBhp3AdTaZ8q4wS5Jvi32V02HNjaC2EsWiEKk= -github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14 h1:+Te28r5Zp4Vp69f82kcON9/BIF8f1BNXb0go2+achuc= -github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14/go.mod h1:Og5KKgoBEiQlI6u56lDLG191pfknIdXctFn3COWLQP8= +github.com/celestiaorg/celestia-app v1.7.0-rc0 h1:vVnUCJAqWAI0aBk+ZWtmWhOF9vsAwQziUnMZSyHn24Q= +github.com/celestiaorg/celestia-app v1.7.0-rc0/go.mod h1:z2H47Gs9gYd3GdQ22d5sbcL8/aBMRcVDtUWT64goMaY= +github.com/celestiaorg/celestia-core v1.35.0-tm-v0.34.29 h1:sXERzNXgyHyqTKNQx4S29C/NMDzgav62DaQDNF49HUQ= +github.com/celestiaorg/celestia-core v1.35.0-tm-v0.34.29/go.mod h1:weZR4wYx1Vcw3g1Jc5G8VipG4M+KUDSqeIzyyWszmsQ= +github.com/celestiaorg/cosmos-sdk v1.20.1-sdk-v0.46.16 h1:9U9UthIJSOyVjabD5PkD6aczvqlWOyAFTOXw0duPT5k= +github.com/celestiaorg/cosmos-sdk v1.20.1-sdk-v0.46.16/go.mod h1:Tvsc3YnqvflXTYC8xIy/Q07Es95xZ1pZC/imoKogtbg= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJx5/hgZ+IeOLEIoLsAveJN/7/ZtQQtPSVw= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= @@ -579,8 +579,8 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:Htrtb github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM= -github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= +github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= +github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -606,8 +606,8 @@ github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakR github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= github.com/etclabscore/go-openrpc-reflect v0.0.37 h1:IH0e7JqIvR9OhbbFWi/BHIkXrqbR3Zyia3RJ733eT6c= github.com/etclabscore/go-openrpc-reflect v0.0.37/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= -github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= -github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= +github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= github.com/ethereum/go-ethereum v1.13.2 h1:g9mCpfPWqCA1OL4e6C98PeVttb0HadfBRuKTGvMnOvw= github.com/ethereum/go-ethereum v1.13.2/go.mod h1:gkQ5Ygi64ZBh9M/4iXY1R8WqoNCx1Ey0CkYn2BD4/fw= @@ -618,8 +618,9 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqL github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -688,8 +689,9 @@ github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBj github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -918,6 +920,12 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8= +github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls= +github.com/grafana/pyroscope-go v1.1.1 h1:PQoUU9oWtO3ve/fgIiklYuGilvsm8qaGhlY4Vw6MAcQ= +github.com/grafana/pyroscope-go v1.1.1/go.mod h1:Mw26jU7jsL/KStNSGGuuVYdUq7Qghem5P8aXYXSXG88= +github.com/grafana/pyroscope-go/godeltaprof v0.1.6 h1:nEdZ8louGAplSvIJi1HVp7kWvFvdiiYg3COLlTwJiFo= +github.com/grafana/pyroscope-go/godeltaprof v0.1.6/go.mod h1:Tk376Nbldo4Cha9RgiU7ik8WKFkNpfds98aUzS8omLE= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -973,8 +981,9 @@ github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoD github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -1341,6 +1350,7 @@ github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -1681,8 +1691,8 @@ github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -1998,8 +2008,6 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= @@ -2131,8 +2139,8 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/rollkit/go-da v0.4.0 h1:/s7ZrVq7DC2aK8UXIvB7rsXrZ2mVGRw7zrexcxRvhlw= github.com/rollkit/go-da v0.4.0/go.mod h1:Kef0XI5ecEKd3TXzI8S+9knAUJnZg0svh2DuXoCsPlM= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= -github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= +github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= @@ -2206,8 +2214,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= -github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= @@ -2223,8 +2231,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= -github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= @@ -2251,8 +2259,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= -github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -2400,6 +2408,7 @@ go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzox go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.24.0 h1:mM8nKi6/iFQ0iqst80wDHU2ge198Ye/TfN0WBS5U24Y= @@ -2412,10 +2421,12 @@ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 h1:VhlEQAPp9R1ktYf go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0/go.mod h1:kB3ufRbfU+CQ4MlUcqtW8Z7YEOBeK2DJ6CmR5rYYF3E= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8= @@ -2424,6 +2435,7 @@ go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16g go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -2872,6 +2884,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= From f09dca68e1133fdc9e3b823a2936a8c7938e3426 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Fri, 1 Mar 2024 13:17:36 -0500 Subject: [PATCH 04/25] chore: bump to Go 1.22 --- Dockerfile | 18 +++++++++--------- README.md | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 01fccafe2e..66fbe6ae7a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM docker.io/golang:1.21-alpine3.18 as builder +FROM --platform=$BUILDPLATFORM docker.io/golang:1.22-alpine3.18 as builder ARG TARGETPLATFORM ARG BUILDPLATFORM @@ -40,16 +40,16 @@ ENV P2P_NETWORK mocha # hadolint ignore=DL3018 RUN uname -a &&\ apk update && apk add --no-cache \ - bash \ - curl \ - jq \ + bash \ + curl \ + jq \ # Creates a user with $UID and $GID=$UID && adduser ${USER_NAME} \ - -D \ - -g ${USER_NAME} \ - -h ${CELESTIA_HOME} \ - -s /sbin/nologin \ - -u ${UID} + -D \ + -g ${USER_NAME} \ + -h ${CELESTIA_HOME} \ + -s /sbin/nologin \ + -u ${UID} # Copy in the binary COPY --from=builder /src/build/celestia /bin/celestia diff --git a/README.md b/README.md index 3a552495ad..423ec17615 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Continue reading [here](https://blog.celestia.org/celestia-mvp-release-data-avai | Requirement | Notes | | ----------- |----------------| -| Go version | 1.21 or higher | +| Go version | 1.22 or higher | ## System Requirements From 0252f7374a714bc2b07c4232c35bacc5f56488a4 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Fri, 1 Mar 2024 13:20:07 -0500 Subject: [PATCH 05/25] ci: bump to Go 1.22 --- .github/workflows/ci_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 73572e4cef..3b95c42b64 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest env: # upgrade go version throughout pipeline here - GO_VERSION: "1.21" + GO_VERSION: "1.22" outputs: go-version: ${{ steps.set-vars.outputs.go-version }} branch: ${{ steps.trim_ref.outputs.branch }} From ebf807876dca8f60ce37a1a8b580b6d0fa99a647 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Fri, 1 Mar 2024 13:33:13 -0500 Subject: [PATCH 06/25] chore: bump pyroscope deps --- cmd/cel-shed/eds_store_stress.go | 2 +- cmd/flags_misc.go | 2 +- go.mod | 7 ++----- go.sum | 9 --------- nodebuilder/settings.go | 4 ++-- 5 files changed, 6 insertions(+), 18 deletions(-) diff --git a/cmd/cel-shed/eds_store_stress.go b/cmd/cel-shed/eds_store_stress.go index 62ea5cb772..eb9d11af29 100644 --- a/cmd/cel-shed/eds_store_stress.go +++ b/cmd/cel-shed/eds_store_stress.go @@ -10,9 +10,9 @@ import ( "os" "time" + "github.com/grafana/pyroscope-go" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" - "github.com/pyroscope-io/client/pyroscope" "github.com/spf13/cobra" "github.com/celestiaorg/celestia-node/libs/edssser" diff --git a/cmd/flags_misc.go b/cmd/flags_misc.go index cd539bde4c..332d2f8022 100644 --- a/cmd/flags_misc.go +++ b/cmd/flags_misc.go @@ -7,8 +7,8 @@ import ( "net/http/pprof" "strings" + otelpyroscope "github.com/grafana/otel-profiling-go" logging "github.com/ipfs/go-log/v2" - otelpyroscope "github.com/pyroscope-io/otel-profiling-go" "github.com/spf13/cobra" flag "github.com/spf13/pflag" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" diff --git a/go.mod b/go.mod index a804c92778..71fc827a50 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,8 @@ require ( github.com/gogo/protobuf v1.3.3 github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.1 + github.com/grafana/otel-profiling-go v0.5.1 + github.com/grafana/pyroscope-go v1.1.1 github.com/hashicorp/go-retryablehttp v0.7.5 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/imdario/mergo v0.3.16 @@ -50,8 +52,6 @@ require ( github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/prometheus/client_golang v1.18.0 - github.com/pyroscope-io/client v0.7.2 - github.com/pyroscope-io/otel-profiling-go v0.5.0 github.com/rollkit/go-da v0.4.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 @@ -178,8 +178,6 @@ require ( github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect - github.com/grafana/otel-profiling-go v0.5.1 // indirect - github.com/grafana/pyroscope-go v1.1.1 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect @@ -279,7 +277,6 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/pyroscope-io/godeltaprof v0.1.2 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-20 v0.4.1 // indirect github.com/quic-go/quic-go v0.40.1 // indirect diff --git a/go.sum b/go.sum index a5e321b654..bdd93f1f06 100644 --- a/go.sum +++ b/go.sum @@ -2093,12 +2093,6 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/pyroscope-io/client v0.7.2 h1:OX2qdUQsS8RSkn/3C8isD7f/P0YiZQlRbAlecAaj/R8= -github.com/pyroscope-io/client v0.7.2/go.mod h1:FEocnjn+Ngzxy6EtU9ZxXWRvQ0+pffkrBxHLnPpxwi8= -github.com/pyroscope-io/godeltaprof v0.1.2 h1:MdlEmYELd5w+lvIzmZvXGNMVzW2Qc9jDMuJaPOR75g4= -github.com/pyroscope-io/godeltaprof v0.1.2/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= -github.com/pyroscope-io/otel-profiling-go v0.5.0 h1:LsTP9VuQ5TgeSiyY2gPHy1de/q3jbFyGWE1v3LtHzMk= -github.com/pyroscope-io/otel-profiling-go v0.5.0/go.mod h1:jUUUXTTgntvGJKS8p5uzypXwTyuGnQP31VnWauH/lUg= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= @@ -2407,7 +2401,6 @@ go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0/go.mod h1:ch3a5QxOqV go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= -go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= @@ -2420,7 +2413,6 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 h1:VhlEQAPp9R1ktYfrPk5SOryw1e9LDDTZCbIPFrho0ec= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0/go.mod h1:kB3ufRbfU+CQ4MlUcqtW8Z7YEOBeK2DJ6CmR5rYYF3E= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= @@ -2434,7 +2426,6 @@ go.opentelemetry.io/otel/sdk/metric v1.24.0/go.mod h1:I6Y5FjH6rvEnTTAYQz3Mmv2kl6 go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= -go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 298976fda4..0e528d8166 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -7,8 +7,8 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/peer" - "github.com/pyroscope-io/client/pyroscope" - otelpyroscope "github.com/pyroscope-io/otel-profiling-go" + "github.com/grafana/pyroscope-go" + otelpyroscope "github.com/grafana/otel-profiling-go" "go.opentelemetry.io/contrib/instrumentation/runtime" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" From d1c8b4045a8cacc9a925b884b62820b148db776c Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Fri, 1 Mar 2024 13:45:19 -0500 Subject: [PATCH 07/25] gofmt -s -w nodebuilder/settings.go --- nodebuilder/settings.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 0e528d8166..7830f0e8f6 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -5,10 +5,10 @@ import ( "fmt" "time" + otelpyroscope "github.com/grafana/otel-profiling-go" + "github.com/grafana/pyroscope-go" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/peer" - "github.com/grafana/pyroscope-go" - otelpyroscope "github.com/grafana/otel-profiling-go" "go.opentelemetry.io/contrib/instrumentation/runtime" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" From e989ead8dc933281ed58f390dd557a329ae78913 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Fri, 1 Mar 2024 13:55:57 -0500 Subject: [PATCH 08/25] chore(deps): upgrade to app v1.7.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 71fc827a50..1f153dbd16 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.7.0-rc0 + github.com/celestiaorg/celestia-app v1.7.0 github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.5.3 github.com/celestiaorg/go-libp2p-messenger v0.2.0 diff --git a/go.sum b/go.sum index bdd93f1f06..04fa9d2c6a 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.7.0-rc0 h1:vVnUCJAqWAI0aBk+ZWtmWhOF9vsAwQziUnMZSyHn24Q= -github.com/celestiaorg/celestia-app v1.7.0-rc0/go.mod h1:z2H47Gs9gYd3GdQ22d5sbcL8/aBMRcVDtUWT64goMaY= +github.com/celestiaorg/celestia-app v1.7.0 h1:pFIgQzeMD036ulJ2ShUR/3bDGG6kwNAWaLEE0MXHmgg= +github.com/celestiaorg/celestia-app v1.7.0/go.mod h1:z2H47Gs9gYd3GdQ22d5sbcL8/aBMRcVDtUWT64goMaY= github.com/celestiaorg/celestia-core v1.35.0-tm-v0.34.29 h1:sXERzNXgyHyqTKNQx4S29C/NMDzgav62DaQDNF49HUQ= github.com/celestiaorg/celestia-core v1.35.0-tm-v0.34.29/go.mod h1:weZR4wYx1Vcw3g1Jc5G8VipG4M+KUDSqeIzyyWszmsQ= github.com/celestiaorg/cosmos-sdk v1.20.1-sdk-v0.46.16 h1:9U9UthIJSOyVjabD5PkD6aczvqlWOyAFTOXw0duPT5k= From c0079bd0688790f25310097d0e463cf8a30d3cb2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Mar 2024 11:32:11 +0000 Subject: [PATCH 09/25] chore(deps): Bump golang.org/x/crypto from 0.19.0 to 0.20.0 (#3234) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.19.0 to 0.20.0.
Commits
  • 0aab8d0 all: update go.mod x/net dependency
  • 5bead59 ocsp: don't use iota for externally defined constants
  • 1a86580 x/crypto/internal/poly1305: improve sum_ppc64le.s
  • 1c981e6 ssh/test: don't use DSA keys in integrations tests, update test RSA key
  • 62c9f17 x509roots/nss: manually exclude a confusingly constrained root
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/crypto&package-manager=go_modules&previous-version=0.19.0&new-version=0.20.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 1f153dbd16..187cff193d 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( go.opentelemetry.io/proto/otlp v1.1.0 go.uber.org/fx v1.20.1 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.19.0 + golang.org/x/crypto v0.20.0 golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 @@ -319,7 +319,7 @@ require ( go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.20.0 // indirect + golang.org/x/net v0.21.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/term v0.17.0 // indirect diff --git a/go.sum b/go.sum index 04fa9d2c6a..4e097dc45a 100644 --- a/go.sum +++ b/go.sum @@ -2527,8 +2527,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= +golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2681,8 +2681,8 @@ golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= From f49b06deebe8d33035ff0131e5f13446dd9feb51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Mar 2024 11:41:05 +0000 Subject: [PATCH 10/25] chore(deps): Bump codecov/codecov-action from 4.0.2 to 4.1.0 (#3236) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4.0.2 to 4.1.0.
Release notes

Sourced from codecov/codecov-action's releases.

v4.1.0

What's Changed

Full Changelog: https://github.com/codecov/codecov-action/compare/v4.0.2...v4.1.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=codecov/codecov-action&package-manager=github_actions&previous-version=4.0.2&new-version=4.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/go-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 0098c3d18a..8c1b8a3d53 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -98,7 +98,7 @@ jobs: retention-days: 5 - name: upload coverage - uses: codecov/codecov-action@v4.0.2 + uses: codecov/codecov-action@v4.1.0 with: env_vars: OS token: ${{ secrets.CODECOV_TOKEN }} From a9c33a670460e3d5647a6037bc316819631857bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Mar 2024 12:16:52 +0000 Subject: [PATCH 11/25] chore(deps): Bump github.com/stretchr/testify from 1.8.4 to 1.9.0 (#3231) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.4 to 1.9.0.
Release notes

Sourced from github.com/stretchr/testify's releases.

v1.9.0

What's Changed

... (truncated)

Commits
  • bb548d0 Merge pull request #1552 from stretchr/dependabot/go_modules/github.com/stret...
  • 814075f build(deps): bump github.com/stretchr/objx from 0.5.1 to 0.5.2
  • e045612 Merge pull request #1339 from bogdandrutu/uintptr
  • 5b6926d Merge pull request #1385 from hslatman/not-implements
  • 9f97d67 Merge pull request #1550 from stretchr/release-notes
  • bcb0d3f Include the auto-release notes in releases
  • fb770f8 Merge pull request #1247 from ccoVeille/typos
  • 85d8bb6 fix typos in comments, tests and github templates
  • e2741fa Merge pull request #1548 from arjunmahishi/msgAndArgs
  • 6e59f20 http_assertions: assert that the msgAndArgs actually works in tests
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/stretchr/testify&package-manager=go_modules&previous-version=1.8.4&new-version=1.9.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 187cff193d..9b4a5a6edd 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/rollkit/go-da v0.4.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/tendermint/tendermint v0.34.29 go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 go.opentelemetry.io/otel v1.24.0 diff --git a/go.sum b/go.sum index 4e097dc45a..a561091a78 100644 --- a/go.sum +++ b/go.sum @@ -2237,8 +2237,9 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -2251,8 +2252,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= From a487efd226f4883d070982d9f20e5b6cfe70ae13 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 5 Mar 2024 13:30:14 +0100 Subject: [PATCH 12/25] deps: bump go-header (#3218) routine --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9b4a5a6edd..249647c700 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/benbjohnson/clock v1.3.5 github.com/celestiaorg/celestia-app v1.7.0 github.com/celestiaorg/go-fraud v0.2.0 - github.com/celestiaorg/go-header v0.5.3 + github.com/celestiaorg/go-header v0.5.4 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.20.0 github.com/celestiaorg/rsmt2d v0.11.0 diff --git a/go.sum b/go.sum index a561091a78..efaef1deeb 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJ github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= -github.com/celestiaorg/go-header v0.5.3 h1:8CcflT6aIlcQXKNWcMekoBNs3EU50mEmDp17gbn1pP4= -github.com/celestiaorg/go-header v0.5.3/go.mod h1:7BVR6myjRfACbqW1de6s8OjuK66XzHm8MpFNYr0G+nU= +github.com/celestiaorg/go-header v0.5.4 h1:7okm3N17/kE5VanlIHv7+bWh5TNAkJxVoNULsYnphAM= +github.com/celestiaorg/go-header v0.5.4/go.mod h1:7BVR6myjRfACbqW1de6s8OjuK66XzHm8MpFNYr0G+nU= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= From f843183221a6ed65911a6a1b7191bdcde620a63b Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 5 Mar 2024 13:42:58 +0100 Subject: [PATCH 13/25] feat(header/headertest): Enable block time intervals for header generation (#3139) Header test suite can now generate headers with the given block time interval if it is specified. Needed for testing pruner service. --- header/headertest/testing.go | 24 +++++++++++++++++------- header/headertest/verify_test.go | 4 ++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/header/headertest/testing.go b/header/headertest/testing.go index e97f7f7825..7b0ae64262 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -36,19 +36,24 @@ type TestSuite struct { valPntr int head *header.ExtendedHeader + + // blockTime is optional - if set, the test suite will generate + // blocks timestamped at the specified interval + blockTime time.Duration } func NewStore(t *testing.T) libhead.Store[*header.ExtendedHeader] { - return headertest.NewStore[*header.ExtendedHeader](t, NewTestSuite(t, 3), 10) + return headertest.NewStore[*header.ExtendedHeader](t, NewTestSuite(t, 3, 0), 10) } // NewTestSuite setups a new test suite with a given number of validators. -func NewTestSuite(t *testing.T, num int) *TestSuite { - valSet, vals := RandValidatorSet(num, 10) +func NewTestSuite(t *testing.T, numValidators int, blockTime time.Duration) *TestSuite { + valSet, vals := RandValidatorSet(numValidators, 10) return &TestSuite{ - t: t, - vals: vals, - valSet: valSet, + t: t, + vals: vals, + valSet: valSet, + blockTime: blockTime, } } @@ -155,13 +160,18 @@ func (s *TestSuite) GenRawHeader( height uint64, lastHeader, lastCommit, dataHash libhead.Hash) *header.RawHeader { rh := RandRawHeader(s.t) rh.Height = int64(height) - rh.Time = time.Now() rh.LastBlockID = types.BlockID{Hash: bytes.HexBytes(lastHeader)} rh.LastCommitHash = bytes.HexBytes(lastCommit) rh.DataHash = bytes.HexBytes(dataHash) rh.ValidatorsHash = s.valSet.Hash() rh.NextValidatorsHash = s.valSet.Hash() rh.ProposerAddress = s.nextProposer().Address + + rh.Time = time.Now() + if s.blockTime > 0 { + rh.Time = s.Head().Time().Add(s.blockTime) + } + return rh } diff --git a/header/headertest/verify_test.go b/header/headertest/verify_test.go index 82795ca5ff..c55d810593 100644 --- a/header/headertest/verify_test.go +++ b/header/headertest/verify_test.go @@ -12,7 +12,7 @@ import ( ) func TestVerify(t *testing.T) { - h := NewTestSuite(t, 2).GenExtendedHeaders(3) + h := NewTestSuite(t, 2, 0).GenExtendedHeaders(3) trusted, untrustedAdj, untrustedNonAdj := h[0], h[1], h[2] tests := []struct { prepare func() *header.ExtendedHeader @@ -47,7 +47,7 @@ func TestVerify(t *testing.T) { { prepare: func() *header.ExtendedHeader { untrusted := *untrustedNonAdj - untrusted.Commit = NewTestSuite(t, 2).Commit(RandRawHeader(t)) + untrusted.Commit = NewTestSuite(t, 2, 0).Commit(RandRawHeader(t)) return &untrusted }, err: header.ErrVerifyCommitLightTrustingFailed, From 32bc7a973b2f0b730c8ec1c2f7d6be5e5484e85b Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 5 Mar 2024 13:56:07 +0100 Subject: [PATCH 14/25] fix(blob | nodebuilder/da): Revert custom marshaling for `blob.Proof` (#3204) Reverts a change introduced in https://github.com/celestiaorg/celestia-node/pull/3146 that unintentionally broke `blob.Proof` serialisation. Resolves https://github.com/celestiaorg/celestia-node/issues/3173 --- blob/blob.go | 27 --------------------------- nodebuilder/da/service.go | 5 +++-- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/blob/blob.go b/blob/blob.go index dc18669e32..c81d83070e 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -39,33 +39,6 @@ type Proof []*nmt.Proof func (p Proof) Len() int { return len(p) } -func (p Proof) MarshalJSON() ([]byte, error) { - proofs := make([]string, 0, len(p)) - for _, proof := range p { - proofBytes, err := proof.MarshalJSON() - if err != nil { - return nil, err - } - proofs = append(proofs, string(proofBytes)) - } - return json.Marshal(proofs) -} - -func (p *Proof) UnmarshalJSON(b []byte) error { - var proofs []string - if err := json.Unmarshal(b, &proofs); err != nil { - return err - } - for _, proof := range proofs { - var nmtProof nmt.Proof - if err := nmtProof.UnmarshalJSON([]byte(proof)); err != nil { - return err - } - *p = append(*p, &nmtProof) - } - return nil -} - // equal is a temporary method that compares two proofs. // should be removed in BlobService V1. func (p Proof) equal(input Proof) error { diff --git a/nodebuilder/da/service.go b/nodebuilder/da/service.go index b775e10396..ea435a5c36 100644 --- a/nodebuilder/da/service.go +++ b/nodebuilder/da/service.go @@ -3,6 +3,7 @@ package da import ( "context" "encoding/binary" + "encoding/json" "fmt" "strings" @@ -83,7 +84,7 @@ func (s *Service) GetProofs(ctx context.Context, ids []da.ID, namespace da.Names if err != nil { return nil, err } - proofs[i], err = proof.MarshalJSON() + proofs[i], err = json.Marshal(proof) if err != nil { return nil, err } @@ -153,7 +154,7 @@ func (s *Service) Validate( proofs := make([]*blob.Proof, len(ids)) for i, daProof := range daProofs { blobProof := &blob.Proof{} - err := blobProof.UnmarshalJSON(daProof) + err := json.Unmarshal(daProof, blobProof) if err != nil { return nil, err } From dac368d21ff94d71e5d449c65097ffa00acfbc27 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Tue, 5 Mar 2024 16:24:53 -0500 Subject: [PATCH 15/25] refactor: rename chunk to share --- das/doc.go | 2 +- docs/adr/adr-011-blocksync-overhaul-part-1.md | 32 +++++++++---------- docs/adr/adr-012-daser-parallelization.md | 6 ++-- share/ipld/add.go | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/das/doc.go b/das/doc.go index bc67fcc7f3..df60ae0d1c 100644 --- a/das/doc.go +++ b/das/doc.go @@ -2,7 +2,7 @@ Package das contains the most important functionality provided by celestia-node. It contains logic for running data availability sampling (DAS) routines on block headers in the network. DAS is the process of verifying the availability of -block data by sampling chunks or shares of those blocks. +block data by sampling shares of those blocks. Package das can confirm the availability of block data in the network via the Availability interface which is implemented both in `full` and `light` mode. diff --git a/docs/adr/adr-011-blocksync-overhaul-part-1.md b/docs/adr/adr-011-blocksync-overhaul-part-1.md index 2967f696ac..0173e78efb 100644 --- a/docs/adr/adr-011-blocksync-overhaul-part-1.md +++ b/docs/adr/adr-011-blocksync-overhaul-part-1.md @@ -94,7 +94,7 @@ are kept with additional components introduced. Altogether, existing and new com foundation of our improved block storage subsystem. The central data structure representing Celestia block data is EDS(`rsmt2d.ExtendedDataSquare`), and the new storage design -is focused around storing entire EDSes as a whole rather than a set of individual chunks, s.t. storage subsystem +is focused around storing entire EDSes as a whole rather than a set of individual shares, s.t. storage subsystem can handle storing and streaming/serving blocks of 4MB and more. #### EDS (De-)Serialization @@ -174,9 +174,9 @@ and getting data by namespace. ```go type Store struct { - basepath string + basepath string dgstr dagstore.DAGStore - topIdx index.Inverted + topIdx index.Inverted carIdx index.FullIndexRepo mounts *mount.Registry ... @@ -189,18 +189,18 @@ func NewStore(basepath string, ds datastore.Batching) *Store { carIdx := index.NewFSRepo(basepath + "/index") mounts := mount.NewRegistry() r := mount.NewRegistry() - err = r.Register("fs", &mount.FSMount{FS: os.DirFS(basePath + "/eds/")}) // registration is a must + err = r.Register("fs", &mount.FSMount{FS: os.DirFS(basePath + "/eds/")}) // registration is a must if err != nil { panic(err) } - + return &Store{ basepath: basepath, dgst: dagstore.New(dagstore.Config{...}), topIdx: index.NewInverted(datastore), carIdx: index.NewFSRepo(basepath + "/index") mounts: mounts, - } + } } ``` @@ -224,7 +224,7 @@ out of the scope of the document._ // // The square is verified on the Exchange level and Put only stores the square trusting it. // The resulting file stores all the shares and NMT Merkle Proofs of the EDS. -// Additionally, the file gets indexed s.t. Store.Blockstore can access them. +// Additionally, the file gets indexed s.t. Store.Blockstore can access them. func (s *Store) Put(context.Context, DataHash, *rsmt2d.ExtendedDataSquare) error ``` @@ -243,9 +243,9 @@ ___NOTES:___ ```go // GetCAR takes a DataHash and returns a buffered reader to the respective EDS serialized as a CARv1 file. -// -// The Reader strictly reads our full EDS, and it's integrity is not verified. -// +// +// The Reader strictly reads our full EDS, and it's integrity is not verified. +// // Caller must Close returned reader after reading. func (s *Store) GetCAR(context.Context, DataHash) (io.ReadCloser, error) ``` @@ -265,8 +265,8 @@ ___NOTES:___ - EDIT: We went with custom implementation. ```go -// Blockstore returns an IPFS Blockstore providing access to individual shares/nodes of all EDS -// registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. +// Blockstore returns an IPFS Blockstore providing access to individual shares/nodes of all EDS +// registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. // We represent `shares` and NMT Merkle proofs as IPFS blocks and IPLD nodes so Bitswap can access those. func (s *Store) Blockstore() blockstore.Blockstore ``` @@ -284,8 +284,8 @@ ___NOTES:___ blocks._ ```go -// CARBlockstore returns an IPFS Blockstore providing access to individual shares/nodes of a specific EDS identified by -// DataHash and registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. +// CARBlockstore returns an IPFS Blockstore providing access to individual shares/nodes of a specific EDS identified by +// DataHash and registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. // We represent `shares` and NMT Merkle proofs as IPFS blocks and IPLD nodes so Bitswap can access those. func (s *Store) CARBlockstore(context.Context, DataHash) (dagstore.ReadBlockstore, error) ``` @@ -301,7 +301,7 @@ The `GetDAH` method returns the DAH (`share.Root`) of the EDS identified by `Dat ```go // GetDAH returns the DataAvailabilityHeader for the EDS identified by DataHash. -func (s *Store) GetDAH(context.Context, share.DataHash) (*share.Root, error) +func (s *Store) GetDAH(context.Context, share.DataHash) (*share.Root, error) ``` ##### `eds.Store.Get` @@ -315,7 +315,7 @@ ___NOTE:___ _It's unnecessary, but an API ergonomics/symmetry nice-to-have._ ```go // Get reads EDS out of Store by given DataHash. -// +// // It reads only one quadrant(1/4) of the EDS and verifies the integrity of the stored data by recomputing it. func (s *Store) Get(context.Context, DataHash) (*rsmt2d.ExtendedDataSquare, error) ``` diff --git a/docs/adr/adr-012-daser-parallelization.md b/docs/adr/adr-012-daser-parallelization.md index a1f3af885b..95e70cee78 100644 --- a/docs/adr/adr-012-daser-parallelization.md +++ b/docs/adr/adr-012-daser-parallelization.md @@ -10,7 +10,7 @@ ## Context -DAS is the process of verifying the availability of block data by sampling chunks or shares of those blocks. The `das` package implements an engine to ensure the availability of the chain's block data via the `Availability` interface. +DAS is the process of verifying the availability of block data by sampling shares of those blocks. The `das` package implements an engine to ensure the availability of the chain's block data via the `Availability` interface. Verifying the availability of block data is a priority functionality for celestia-node. Its performance could benefit significantly from parallelization optimisation to make it able to fully utilise network bandwidth. ## Previous approach @@ -61,7 +61,7 @@ amount of workers: 32, speed: 11.33 amount of workers: 64, speed: 11.83 ``` -Based on basic experiment results, values higher than 16 don’t bring much benefit. At the same time, increased parallelization comes with a cost of higher memory consumption. +Based on basic experiment results, values higher than 16 don’t bring much benefit. At the same time, increased parallelization comes with a cost of higher memory consumption. Future improvements will be discussed later and are out of the scope of this ADR. ## Status @@ -70,7 +70,7 @@ Implemented ## Future plans -Several params values that come hardcoded in DASer (`samplingRange`, `concurrencyLimit`, `priorityQueueSize`, `genesisHeight`, `backgroundStoreInterval`) should become configurable, so the node runner can define them based on the specific node setup. Default values should be optimized by performance testing for most common setups, and could potentially vary for different node types. +Several params values that come hardcoded in DASer (`samplingRange`, `concurrencyLimit`, `priorityQueueSize`, `genesisHeight`, `backgroundStoreInterval`) should become configurable, so the node runner can define them based on the specific node setup. Default values should be optimized by performance testing for most common setups, and could potentially vary for different node types. ## References diff --git a/share/ipld/add.go b/share/ipld/add.go index fbb743148c..4dc5f1cf47 100644 --- a/share/ipld/add.go +++ b/share/ipld/add.go @@ -47,7 +47,7 @@ func AddShares( return eds, batchAdder.Commit() } -// ImportShares imports flattened chunks of data into Extended Data square and saves it in +// ImportShares imports flattened pieces of data into Extended Data square and saves it in // blockservice.BlockService func ImportShares( ctx context.Context, From e9026800ed67859deb0a4f31e832a4586bee1d45 Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 8 Mar 2024 10:09:55 +0000 Subject: [PATCH 16/25] ci(triggers): change release process from tag push to release publish (#3227) Updates the CI workflow trigger to run specifically from a release publish, vs pushing a tag. This will separate any testing we do with with any pre-release or temporary tags from triggering the GitHub workflow to generate a release. Instead, only the release published event should trigger this. --------- Co-authored-by: Matthew Sevey --- .github/workflows/ci_release.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 3b95c42b64..3c17aca2e3 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -4,9 +4,8 @@ on: push: branches: - main - # Trigger on version tags - tags: - - "v*" + release: + types: [published] pull_request: workflow_dispatch: inputs: @@ -111,7 +110,7 @@ jobs: runs-on: ubuntu-latest if: | github.event_name == 'workflow_dispatch' || - (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) + github.event_name == 'release' permissions: "write-all" steps: - uses: actions/checkout@v4 From b0a724ed4734348c1c6cd33f0d801264df9ab28c Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 13 Mar 2024 11:01:37 +0000 Subject: [PATCH 17/25] chore(Makefile): detect and default to make go-install on darwin vs install everywhere else (#3197) refs #3169 Suggestion we add this little detect and switch. One issue i see with attempting to detect GOBIN is that it doesn't HAVE to be set (ie: there can be a default). This i would think is a good quick compromise to un-confuse @rootulp and (potentially) other devs and make this behavior more delightful most of the time. IF not, we should close this and the other issue. --------- Co-authored-by: Rootul P --- Makefile | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a43c917345..ddc0b93a6b 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ SHELL=/usr/bin/env bash PROJECTNAME=$(shell basename "$(PWD)") DIR_FULLPATH=$(shell pwd) versioningPath := "github.com/celestiaorg/celestia-node/nodebuilder/node" +OS := $(shell uname -s) LDFLAGS=-ldflags="-X '$(versioningPath).buildTime=$(shell date)' -X '$(versioningPath).lastCommit=$(shell git rev-parse HEAD)' -X '$(versioningPath).semanticVersion=$(shell git describe --tags --dirty=-dev 2>/dev/null || git rev-parse --abbrev-ref HEAD)'" TAGS=integration SHORT= @@ -63,11 +64,19 @@ deps: @go mod download .PHONY: deps -## install: Install all build binaries into the $PREFIX (/usr/local/ by default) directory. +## install: Install the celestia-node binary. install: +ifeq ($(OS),Darwin) + @$(MAKE) go-install +else + @$(MAKE) install-global +endif +.PHONY: install + +install-global: @echo "--> Installing Celestia" @install -v ./build/* -t ${PREFIX}/bin/ -.PHONY: install +.PHONY: install-global ## go-install: Build and install the celestia-node binary into the GOBIN directory. go-install: From 3b7fd16bfc74d32aada33f74f2ce02be7f185115 Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 13 Mar 2024 11:08:38 +0000 Subject: [PATCH 18/25] ci(lint): upgrade golangci-lint to 1.56 and fix revive errors that emerged (#3247) with our `1.22` upgrade i got inadvertently upgraded to `golangci-lint` 1.56 locally and experiencing different results to what was in CI so figured, why not upgrade, i believe this upgrade makes 'revive' active and a replacement to golint, which introduced some better/stricter checks - upgrades `golangci-lint` to 1.56 - fixes or marks `//nolint:revive` a few new unused variable errors that were triggering - configures revive to ignore `_test` files - CI passes with new checks --- .github/workflows/go-ci.yml | 2 +- .golangci.yml | 1 + api/docgen/openrpc.go | 4 +++- cmd/cel-shed/eds_store_stress.go | 2 +- cmd/cel-shed/p2p.go | 4 ++-- cmd/config.go | 4 ++-- cmd/docgen/openrpc.go | 2 +- cmd/init.go | 2 +- cmd/reset_store.go | 2 +- cmd/start.go | 2 +- nodebuilder/core/module.go | 4 ++-- nodebuilder/das/cmd/das.go | 2 +- nodebuilder/header/cmd/header.go | 6 +++--- nodebuilder/node/cmd/node.go | 2 +- nodebuilder/node/metrics.go | 2 +- nodebuilder/p2p/bitswap.go | 2 +- nodebuilder/p2p/cmd/p2p.go | 10 +++++----- nodebuilder/p2p/metrics.go | 2 +- nodebuilder/share/module.go | 4 ++-- nodebuilder/state/cmd/state.go | 4 ++-- share/eds/cache/metrics.go | 2 +- share/eds/metrics.go | 2 +- share/p2p/discovery/metrics.go | 2 +- share/p2p/peers/metrics.go | 2 +- state/metrics.go | 2 +- 25 files changed, 38 insertions(+), 35 deletions(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 8c1b8a3d53..0bcd33b2f6 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -44,7 +44,7 @@ jobs: uses: golangci/golangci-lint-action@v4.0.0 with: args: --timeout 10m - version: v1.55 + version: v1.56 skip-pkg-cache: true skip-build-cache: true diff --git a/.golangci.yml b/.golangci.yml index a0f2754a9b..bcc34b20f0 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -48,6 +48,7 @@ issues: - path: _test\.go linters: - gosec + - revive - linters: - lll source: "https://" diff --git a/api/docgen/openrpc.go b/api/docgen/openrpc.go index 737b491a89..4ec98d73b0 100644 --- a/api/docgen/openrpc.go +++ b/api/docgen/openrpc.go @@ -1,5 +1,7 @@ // Package docgen generates an OpenRPC spec for the Celestia Node. It has been inspired by and // adapted from Filecoin's Lotus API implementation. + +//nolint:revive package docgen import ( @@ -95,7 +97,7 @@ func NewOpenRPCDocument(comments Comments, permissions Comments) *go_openrpc_ref d.WithMeta(&go_openrpc_reflect.MetaT{ GetServersFn: func() func(listeners []net.Listener) (*meta_schema.Servers, error) { - return func(listeners []net.Listener) (*meta_schema.Servers, error) { + return func(_ []net.Listener) (*meta_schema.Servers, error) { return nil, nil } }, diff --git a/cmd/cel-shed/eds_store_stress.go b/cmd/cel-shed/eds_store_stress.go index eb9d11af29..9036a81e30 100644 --- a/cmd/cel-shed/eds_store_stress.go +++ b/cmd/cel-shed/eds_store_stress.go @@ -69,7 +69,7 @@ var edsStoreStress = &cobra.Command{ Use: "stress", Short: `Runs eds.Store stress test over default node.Store Datastore backend (e.g. Badger).`, SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) (err error) { + RunE: func(cmd *cobra.Command, _ []string) (err error) { // expose expvar vars over http go http.ListenAndServe(":9999", http.DefaultServeMux) //nolint:errcheck,gosec diff --git a/cmd/cel-shed/p2p.go b/cmd/cel-shed/p2p.go index a313841fa9..d4a6c4b596 100644 --- a/cmd/cel-shed/p2p.go +++ b/cmd/cel-shed/p2p.go @@ -22,7 +22,7 @@ var p2pCmd = &cobra.Command{ var p2pNewKeyCmd = &cobra.Command{ Use: "new-key", Short: "Generate and print new Ed25519 private key for p2p networking", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { //nolint:revive privkey, _, err := crypto.GenerateEd25519Key(rand.Reader) if err != nil { return err @@ -42,7 +42,7 @@ var p2pNewKeyCmd = &cobra.Command{ var p2pPeerIDCmd = &cobra.Command{ Use: "peer-id", Short: "Get peer-id out of public or private Ed25519 key", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, args []string) error { //nolint:revive decKey, err := hex.DecodeString(args[0]) if err != nil { return err diff --git a/cmd/config.go b/cmd/config.go index 4a2d322adf..51bba81b45 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -12,7 +12,7 @@ func RemoveConfigCmd(fsets ...*flag.FlagSet) *cobra.Command { Use: "config-remove", Short: "Deletes the node's config", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() return nodebuilder.RemoveConfig(StorePath(ctx)) }, @@ -31,7 +31,7 @@ func UpdateConfigCmd(fsets ...*flag.FlagSet) *cobra.Command { Long: "Updates the node's outdated config with default values from newly-added fields. Check the config " + " afterwards to ensure all old custom values were preserved.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() return nodebuilder.UpdateConfig(NodeType(ctx), StorePath(ctx)) }, diff --git a/cmd/docgen/openrpc.go b/cmd/docgen/openrpc.go index cd5b57a82c..ef8b706ec5 100644 --- a/cmd/docgen/openrpc.go +++ b/cmd/docgen/openrpc.go @@ -14,7 +14,7 @@ import ( var rootCmd = &cobra.Command{ Use: "docgen [packages]", Short: "docgen generates the openrpc documentation for Celestia Node packages", - RunE: func(cmd *cobra.Command, moduleNames []string) error { + RunE: func(_ *cobra.Command, moduleNames []string) error { // 1. Open the respective nodebuilder/X/service.go files for AST parsing nodeComments, permComments := docgen.ParseCommentsFromNodebuilderModules(moduleNames...) diff --git a/cmd/init.go b/cmd/init.go index 5eaa465701..abad602d94 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -13,7 +13,7 @@ func Init(fsets ...*flag.FlagSet) *cobra.Command { Use: "init", Short: "Initialization for Celestia Node. Passed flags have persisted effect.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() return nodebuilder.Init(NodeConfig(ctx), StorePath(ctx), NodeType(ctx)) diff --git a/cmd/reset_store.go b/cmd/reset_store.go index d386549efa..a839d765ab 100644 --- a/cmd/reset_store.go +++ b/cmd/reset_store.go @@ -13,7 +13,7 @@ func ResetStore(fsets ...*flag.FlagSet) *cobra.Command { Use: "unsafe-reset-store", Short: "Resets the node's store to a new state. Leaves the keystore and config intact.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() return nodebuilder.Reset(StorePath(ctx), NodeType(ctx)) diff --git a/cmd/start.go b/cmd/start.go index 281dfcc0e4..c3ee37fe72 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -25,7 +25,7 @@ Options passed on start override configuration options only on start and are not Aliases: []string{"run", "daemon"}, Args: cobra.NoArgs, SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) (err error) { + RunE: func(cmd *cobra.Command, _ []string) (err error) { ctx := cmd.Context() // override config with all modifiers passed on start diff --git a/nodebuilder/core/module.go b/nodebuilder/core/module.go index 7c5c9e6bfd..fcf682cdf5 100644 --- a/nodebuilder/core/module.go +++ b/nodebuilder/core/module.go @@ -74,10 +74,10 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option )), fx.Provide(fx.Annotate( remote, - fx.OnStart(func(ctx context.Context, client core.Client) error { + fx.OnStart(func(_ context.Context, client core.Client) error { return client.Start() }), - fx.OnStop(func(ctx context.Context, client core.Client) error { + fx.OnStop(func(_ context.Context, client core.Client) error { return client.Stop() }), )), diff --git a/nodebuilder/das/cmd/das.go b/nodebuilder/das/cmd/das.go index 7512861ac3..5678190c9c 100644 --- a/nodebuilder/das/cmd/das.go +++ b/nodebuilder/das/cmd/das.go @@ -21,7 +21,7 @@ var samplingStatsCmd = &cobra.Command{ Use: "sampling-stats", Short: "Returns the current statistics over the DA sampling process", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err diff --git a/nodebuilder/header/cmd/header.go b/nodebuilder/header/cmd/header.go index b3bba1eb32..6bcf84d414 100644 --- a/nodebuilder/header/cmd/header.go +++ b/nodebuilder/header/cmd/header.go @@ -31,7 +31,7 @@ var localHeadCmd = &cobra.Command{ Use: "local-head", Short: "Returns the ExtendedHeader from the chain head.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -47,7 +47,7 @@ var networkHeadCmd = &cobra.Command{ Use: "network-head", Short: "Provides the Syncer's view of the current network head.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -104,7 +104,7 @@ var syncStateCmd = &cobra.Command{ Use: "sync-state", Short: "Returns the current state of the header Syncer.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err diff --git a/nodebuilder/node/cmd/node.go b/nodebuilder/node/cmd/node.go index a65727fb03..bef76b9683 100644 --- a/nodebuilder/node/cmd/node.go +++ b/nodebuilder/node/cmd/node.go @@ -25,7 +25,7 @@ var nodeInfoCmd = &cobra.Command{ Use: "info", Args: cobra.NoArgs, Short: "Returns administrative information about the node.", - RunE: func(c *cobra.Command, args []string) error { + RunE: func(c *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(c.Context()) if err != nil { return err diff --git a/nodebuilder/node/metrics.go b/nodebuilder/node/metrics.go index 07c9a5fc0f..560df808e6 100644 --- a/nodebuilder/node/metrics.go +++ b/nodebuilder/node/metrics.go @@ -42,7 +42,7 @@ func WithMetrics() error { return err } - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { if !nodeStarted { // Observe node start timestamp timeStarted = time.Now() diff --git a/nodebuilder/p2p/bitswap.go b/nodebuilder/p2p/bitswap.go index 014435071a..773d39ba4e 100644 --- a/nodebuilder/p2p/bitswap.go +++ b/nodebuilder/p2p/bitswap.go @@ -53,7 +53,7 @@ func dataExchange(params bitSwapParams) exchange.Interface { net.Start(srvr, clnt) // starting with hook does not work params.Lifecycle.Append(fx.Hook{ - OnStop: func(ctx context.Context) (err error) { + OnStop: func(_ context.Context) (err error) { err = errors.Join(err, clnt.Close()) err = errors.Join(err, srvr.Close()) net.Stop() diff --git a/nodebuilder/p2p/cmd/p2p.go b/nodebuilder/p2p/cmd/p2p.go index 8b44802947..64c36fc9d6 100644 --- a/nodebuilder/p2p/cmd/p2p.go +++ b/nodebuilder/p2p/cmd/p2p.go @@ -48,7 +48,7 @@ var infoCmd = &cobra.Command{ Use: "info", Short: "Gets the node's peer info (peer id and multiaddresses)", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -77,7 +77,7 @@ var peersCmd = &cobra.Command{ Use: "peers", Short: "Lists the peers we are connected to", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -226,7 +226,7 @@ var natStatusCmd = &cobra.Command{ Use: "nat-status", Short: "Gets the current NAT status", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -328,7 +328,7 @@ var blockedPeersCmd = &cobra.Command{ Use: "blocked-peers", Short: "Lists the node's blocked peers", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -466,7 +466,7 @@ var bandwidthStatsCmd = &cobra.Command{ Long: "Get stats struct with bandwidth metrics for all data sent/" + "received by the local peer, regardless of protocol or remote peer IDs", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err diff --git a/nodebuilder/p2p/metrics.go b/nodebuilder/p2p/metrics.go index 095c30d9b7..ca226c98ae 100644 --- a/nodebuilder/p2p/metrics.go +++ b/nodebuilder/p2p/metrics.go @@ -41,7 +41,7 @@ func prometheusMetrics(lifecycle fx.Lifecycle, registerer prometheus.Registerer) } lifecycle.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { + OnStart: func(_ context.Context) error { go func() { if err := promHTTPServer.ListenAndServe(); err != nil { log.Errorf("Error starting Prometheus metrics exporter http server: %s", err) diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index 7caaf39a92..b81149535f 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -100,7 +100,7 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option func discoveryComponents(cfg *Config) fx.Option { return fx.Options( - fx.Invoke(func(disc *disc.Discovery) {}), + fx.Invoke(func(_ *disc.Discovery) {}), fx.Provide(fx.Annotate( newDiscovery(cfg.Discovery), fx.OnStart(func(ctx context.Context, d *disc.Discovery) error { @@ -189,7 +189,7 @@ func shrexGetterComponents(cfg *Config) fx.Option { func shrexServerComponents(cfg *Config) fx.Option { return fx.Options( - fx.Invoke(func(edsSrv *shrexeds.Server, ndSrc *shrexnd.Server) {}), + fx.Invoke(func(_ *shrexeds.Server, _ *shrexnd.Server) {}), fx.Provide(fx.Annotate( func(host host.Host, store *eds.Store, network modp2p.Network) (*shrexeds.Server, error) { cfg.ShrExEDSParams.WithNetworkID(network.String()) diff --git a/nodebuilder/state/cmd/state.go b/nodebuilder/state/cmd/state.go index d35c4a1b4f..031c118f98 100644 --- a/nodebuilder/state/cmd/state.go +++ b/nodebuilder/state/cmd/state.go @@ -40,7 +40,7 @@ var accountAddressCmd = &cobra.Command{ Use: "account-address", Short: "Retrieves the address of the node's account/signer.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err @@ -57,7 +57,7 @@ var balanceCmd = &cobra.Command{ Short: "Retrieves the Celestia coin balance for the node's account/signer and verifies it against " + "the corresponding block's AppHash.", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err diff --git a/share/eds/cache/metrics.go b/share/eds/cache/metrics.go index b2e3bec8d8..565a61a5e0 100644 --- a/share/eds/cache/metrics.go +++ b/share/eds/cache/metrics.go @@ -39,7 +39,7 @@ func newMetrics(bc *AccessorCache) (*metrics, error) { return nil, err } - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { observer.ObserveInt64(cacheSize, int64(bc.cache.Len())) return nil } diff --git a/share/eds/metrics.go b/share/eds/metrics.go index 8d69a3ec41..0fd6740154 100644 --- a/share/eds/metrics.go +++ b/share/eds/metrics.go @@ -128,7 +128,7 @@ func (s *Store) WithMetrics() error { return err } - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { stats := s.dgstr.Stats() for status, amount := range stats { observer.ObserveInt64(dagStoreShards, int64(amount), diff --git a/share/p2p/discovery/metrics.go b/share/p2p/discovery/metrics.go index 78b62a7d97..c91e68491d 100644 --- a/share/p2p/discovery/metrics.go +++ b/share/p2p/discovery/metrics.go @@ -102,7 +102,7 @@ func initMetrics(d *Discovery) (*metrics, error) { peerRemoved: peerRemoved, } - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { observer.ObserveInt64(peersAmount, int64(d.set.Size())) observer.ObserveInt64(backOffSize, int64(d.connector.Size())) return nil diff --git a/share/p2p/peers/metrics.go b/share/p2p/peers/metrics.go index eb42254430..094d81a5e3 100644 --- a/share/p2p/peers/metrics.go +++ b/share/p2p/peers/metrics.go @@ -130,7 +130,7 @@ func initMetrics(manager *Manager) (*metrics, error) { blacklistedPeers: blacklisted, } - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { for poolStatus, count := range manager.shrexPools() { observer.ObserveInt64(shrexPools, count, metric.WithAttributes( diff --git a/state/metrics.go b/state/metrics.go index aa166e901d..3672ef9b36 100644 --- a/state/metrics.go +++ b/state/metrics.go @@ -19,7 +19,7 @@ func WithMetrics(ca *CoreAccessor) { metric.WithDescription("Timestamp of the last submitted PayForBlob transaction"), ) - callback := func(ctx context.Context, observer metric.Observer) error { + callback := func(_ context.Context, observer metric.Observer) error { observer.ObserveInt64(pfbCounter, ca.PayForBlobCount()) observer.ObserveInt64(lastPfbTimestamp, ca.LastPayForBlob()) return nil From 3b279bcfcefad8ebd968ced5959cd2f5da3a5075 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Wed, 13 Mar 2024 14:28:57 +0200 Subject: [PATCH 19/25] fix(nodebuilder/blob): use state module instead of the state accessor in the module construction (#3254) Resolves #3244 --- nodebuilder/blob/module.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodebuilder/blob/module.go b/nodebuilder/blob/module.go index 76e7677725..56a6c906e9 100644 --- a/nodebuilder/blob/module.go +++ b/nodebuilder/blob/module.go @@ -8,8 +8,8 @@ import ( "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/header" headerService "github.com/celestiaorg/celestia-node/nodebuilder/header" + "github.com/celestiaorg/celestia-node/nodebuilder/state" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/state" ) func ConstructModule() fx.Option { @@ -19,7 +19,7 @@ func ConstructModule() fx.Option { return service.GetByHeight }), fx.Provide(func( - state *state.CoreAccessor, + state state.Module, sGetter share.Getter, getByHeightFn func(context.Context, uint64) (*header.ExtendedHeader, error), ) Module { From 648f61a809ee1f9b65fb9928f8c3ae2065ccff7f Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 13 Mar 2024 14:01:37 +0100 Subject: [PATCH 20/25] refactor(nodebuilder): use external flock library (#3253) This PR is prompted by #3246. While looking through the code and understanding how `IsLocked` worked there, I realized that our fslock lib was missing a feature. Instead of asking @mastergaurang94 to fix it, I decided to completely migrate over to a more [comprehensive locking solution](https://github.com/gofrs/flock), which additionally: * avoids the need custom code to maintain(NIHS) * solves the problem of locking being annoying after an ungraceful shutdown * gives us Windows and other niche OS support * as well as works around niche FS edge-cases --- go.mod | 1 + libs/fslock/lock_unix.go | 48 ------------------------------------- libs/fslock/locker.go | 49 -------------------------------------- libs/fslock/locker_test.go | 40 ------------------------------- nodebuilder/config.go | 29 ++++++++++++---------- nodebuilder/init.go | 28 ++++++++++++---------- nodebuilder/init_test.go | 7 +++--- nodebuilder/store.go | 29 +++++++++++----------- 8 files changed, 51 insertions(+), 180 deletions(-) delete mode 100644 libs/fslock/lock_unix.go delete mode 100644 libs/fslock/locker.go delete mode 100644 libs/fslock/locker_test.go diff --git a/go.mod b/go.mod index 249647c700..913caa91df 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/filecoin-project/dagstore v0.5.6 github.com/filecoin-project/go-jsonrpc v0.3.1 github.com/gammazero/workerpool v1.1.3 + github.com/gofrs/flock v0.8.1 github.com/gogo/protobuf v1.3.3 github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.1 diff --git a/libs/fslock/lock_unix.go b/libs/fslock/lock_unix.go deleted file mode 100644 index bbb8db8537..0000000000 --- a/libs/fslock/lock_unix.go +++ /dev/null @@ -1,48 +0,0 @@ -//go:build darwin || freebsd || linux - -package fslock - -import ( - "fmt" - "os" - "strconv" - "syscall" -) - -func (l *Locker) lock() (err error) { - l.file, err = os.OpenFile(l.path, os.O_CREATE|os.O_RDWR, 0666) - if err != nil { - return fmt.Errorf("fslock: error opening file: %w", err) - } - - _, err = l.file.WriteString(strconv.Itoa(os.Getpid())) - if err != nil { - return fmt.Errorf("fslock: error writing process id: %w", err) - } - - err = syscall.Flock(int(l.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) - if err != nil && err.Error() == "resource temporarily unavailable" { - return ErrLocked // we have to check here for a string in err, as there is no error types defined for this case. - } - if err != nil { - return fmt.Errorf("fslock: flocking error: %w", err) - } - - return -} - -func (l *Locker) unlock() error { - err := syscall.Flock(int(l.file.Fd()), syscall.LOCK_UN|syscall.LOCK_NB) - if err != nil { - return fmt.Errorf("fslock: unflocking error: %w", err) - } - - file := l.file - l.file = nil - err = file.Close() - if err != nil { - return fmt.Errorf("fslock: while closing file: %w", err) - } - - return os.Remove(l.path) -} diff --git a/libs/fslock/locker.go b/libs/fslock/locker.go deleted file mode 100644 index f451c42cc1..0000000000 --- a/libs/fslock/locker.go +++ /dev/null @@ -1,49 +0,0 @@ -package fslock - -import ( - "errors" - "os" -) - -// ErrLocked is signaled when someone tries to lock an already locked file. -var ErrLocked = errors.New("fslock: directory is locked") - -// Lock creates a new Locker under the given 'path' -// and immediately does Lock on it. -func Lock(path string) (*Locker, error) { - l := New(path) - err := l.Lock() - if err != nil { - return nil, err - } - - return l, nil -} - -// Locker is a simple utility meant to create lock files. -// This is to prevent multiple processes from managing the same working directory by purpose or -// accident. NOTE: Windows is not supported. -type Locker struct { - file *os.File - path string -} - -// New creates a new Locker with a File pointing to the given 'path'. -func New(path string) *Locker { - return &Locker{path: path} -} - -// Lock locks the file. -// Subsequent calls will error with ErrLocked on any Locker instance looking to the same path. -func (l *Locker) Lock() error { - return l.lock() -} - -// Unlock frees up the lock. -func (l *Locker) Unlock() error { - if l == nil || l.file == nil { - return nil - } - - return l.unlock() -} diff --git a/libs/fslock/locker_test.go b/libs/fslock/locker_test.go deleted file mode 100644 index c80ce51c16..0000000000 --- a/libs/fslock/locker_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package fslock - -import ( - "os" - "path/filepath" - "testing" -) - -func TestLocker(t *testing.T) { - path := filepath.Join(os.TempDir(), ".lock") - defer os.Remove(path) - - locker := New(path) - locker2 := New(path) - - err := locker.Lock() - if err != nil { - t.Fatal(err) - } - - err = locker2.Lock() - if err != ErrLocked { - t.Fatal("No locking") - } - - err = locker.Unlock() - if err != nil { - t.Fatal(err) - } - - err = locker2.Lock() - if err != nil { - t.Fatal(err) - } - - err = locker2.Unlock() - if err != nil { - t.Fatal(err) - } -} diff --git a/nodebuilder/config.go b/nodebuilder/config.go index 41f24d6d3d..03f3124093 100644 --- a/nodebuilder/config.go +++ b/nodebuilder/config.go @@ -1,13 +1,14 @@ package nodebuilder import ( + "fmt" "io" "os" "github.com/BurntSushi/toml" + "github.com/gofrs/flock" "github.com/imdario/mergo" - "github.com/celestiaorg/celestia-node/libs/fslock" "github.com/celestiaorg/celestia-node/nodebuilder/core" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/gateway" @@ -91,14 +92,15 @@ func RemoveConfig(path string) (err error) { return } - flock, err := fslock.Lock(lockPath(path)) + flk := flock.New(lockPath(path)) + ok, err := flk.TryLock() if err != nil { - if err == fslock.ErrLocked { - err = ErrOpened - } - return + return fmt.Errorf("locking file: %w", err) + } + defer flk.Close() + if !ok { + return ErrOpened } - defer flock.Unlock() //nolint: errcheck return removeConfig(configPath(path)) } @@ -117,14 +119,15 @@ func UpdateConfig(tp node.Type, path string) (err error) { return err } - flock, err := fslock.Lock(lockPath(path)) + flk := flock.New(lockPath(path)) + ok, err := flk.TryLock() if err != nil { - if err == fslock.ErrLocked { - err = ErrOpened - } - return err + return fmt.Errorf("locking file: %w", err) + } + defer flk.Close() + if !ok { + return ErrOpened } - defer flock.Unlock() //nolint: errcheck newCfg := DefaultConfig(tp) diff --git a/nodebuilder/init.go b/nodebuilder/init.go index 0593d88560..4604a4f7ee 100644 --- a/nodebuilder/init.go +++ b/nodebuilder/init.go @@ -8,11 +8,11 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/hd" "github.com/cosmos/cosmos-sdk/crypto/keyring" sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/gofrs/flock" "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" - "github.com/celestiaorg/celestia-node/libs/fslock" "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/state" @@ -35,14 +35,15 @@ func Init(cfg Config, path string, tp node.Type) error { return err } - flock, err := fslock.Lock(lockPath(path)) + flk := flock.New(lockPath(path)) + ok, err := flk.TryLock() if err != nil { - if err == fslock.ErrLocked { - return ErrOpened - } - return err + return fmt.Errorf("locking file: %w", err) + } + defer flk.Close() + if !ok { + return ErrOpened } - defer flock.Unlock() //nolint: errcheck ksPath := keysPath(path) err = initDir(ksPath) @@ -82,14 +83,15 @@ func Reset(path string, tp node.Type) error { } log.Infof("Resetting %s Node Store over '%s'", tp, path) - flock, err := fslock.Lock(lockPath(path)) + flk := flock.New(lockPath(path)) + ok, err := flk.TryLock() if err != nil { - if err == fslock.ErrLocked { - return ErrOpened - } - return err + return fmt.Errorf("locking file: %w", err) + } + defer flk.Close() + if !ok { + return ErrOpened } - defer flock.Unlock() //nolint: errcheck err = resetDir(dataPath(path)) if err != nil { diff --git a/nodebuilder/init_test.go b/nodebuilder/init_test.go index e438a191bc..fffe2517e8 100644 --- a/nodebuilder/init_test.go +++ b/nodebuilder/init_test.go @@ -7,13 +7,13 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/hd" "github.com/cosmos/cosmos-sdk/crypto/keyring" sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/gofrs/flock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" - "github.com/celestiaorg/celestia-node/libs/fslock" "github.com/celestiaorg/celestia-node/nodebuilder/node" ) @@ -58,9 +58,10 @@ func TestIsInitForNonExistDir(t *testing.T) { func TestInitErrForLockedDir(t *testing.T) { dir := t.TempDir() - flock, err := fslock.Lock(lockPath(dir)) + flk := flock.New(lockPath(dir)) + _, err := flk.TryLock() require.NoError(t, err) - defer flock.Unlock() //nolint:errcheck + defer flk.Close() nodes := []node.Type{node.Light, node.Bridge} for _, node := range nodes { diff --git a/nodebuilder/store.go b/nodebuilder/store.go index 7f67a9e782..e5c051640c 100644 --- a/nodebuilder/store.go +++ b/nodebuilder/store.go @@ -10,11 +10,11 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/keyring" "github.com/dgraph-io/badger/v4/options" + "github.com/gofrs/flock" "github.com/ipfs/go-datastore" dsbadger "github.com/ipfs/go-ds-badger4" "github.com/mitchellh/go-homedir" - "github.com/celestiaorg/celestia-node/libs/fslock" "github.com/celestiaorg/celestia-node/libs/keystore" "github.com/celestiaorg/celestia-node/share" ) @@ -58,28 +58,29 @@ func OpenStore(path string, ring keyring.Keyring) (Store, error) { return nil, err } - flock, err := fslock.Lock(lockPath(path)) + flk := flock.New(lockPath(path)) + ok, err := flk.TryLock() if err != nil { - if err == fslock.ErrLocked { - return nil, ErrOpened - } - return nil, err + return nil, fmt.Errorf("locking file: %w", err) } - - ok := IsInit(path) if !ok { - flock.Unlock() //nolint:errcheck - return nil, ErrNotInited + return nil, ErrOpened + } + + if !IsInit(path) { + err := errors.Join(ErrNotInited, flk.Close()) + return nil, err } ks, err := keystore.NewFSKeystore(keysPath(path), ring) if err != nil { + err = errors.Join(err, flk.Close()) return nil, err } return &fsStore{ path: path, - dirLock: flock, + dirLock: flk, keys: ks, }, nil } @@ -131,7 +132,7 @@ func (f *fsStore) Datastore() (datastore.Batching, error) { } func (f *fsStore) Close() (err error) { - err = errors.Join(err, f.dirLock.Unlock()) + err = errors.Join(err, f.dirLock.Close()) f.dataMu.Lock() if f.data != nil { err = errors.Join(err, f.data.Close()) @@ -146,7 +147,7 @@ type fsStore struct { dataMu sync.Mutex data datastore.Batching keys keystore.Keystore - dirLock *fslock.Locker // protects directory + dirLock *flock.Flock // protects directory } func storePath(path string) (string, error) { @@ -158,7 +159,7 @@ func configPath(base string) string { } func lockPath(base string) string { - return filepath.Join(base, "lock") + return filepath.Join(base, ".lock") } func keysPath(base string) string { From 4b0586cc8a405789bdc81156782ab9542193f807 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 13 Mar 2024 14:40:55 +0100 Subject: [PATCH 21/25] refactor(nodebuilder): stick to lock semantics, not file (#3255) Review discussion that did't get to the PR because of automerge --- nodebuilder/config.go | 4 ++-- nodebuilder/init.go | 4 ++-- nodebuilder/init_test.go | 2 +- nodebuilder/store.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nodebuilder/config.go b/nodebuilder/config.go index 03f3124093..d323f401d7 100644 --- a/nodebuilder/config.go +++ b/nodebuilder/config.go @@ -97,10 +97,10 @@ func RemoveConfig(path string) (err error) { if err != nil { return fmt.Errorf("locking file: %w", err) } - defer flk.Close() if !ok { return ErrOpened } + defer flk.Unlock() //nolint:errcheck return removeConfig(configPath(path)) } @@ -124,10 +124,10 @@ func UpdateConfig(tp node.Type, path string) (err error) { if err != nil { return fmt.Errorf("locking file: %w", err) } - defer flk.Close() if !ok { return ErrOpened } + defer flk.Unlock() //nolint:errcheck newCfg := DefaultConfig(tp) diff --git a/nodebuilder/init.go b/nodebuilder/init.go index 4604a4f7ee..fb3aeedf02 100644 --- a/nodebuilder/init.go +++ b/nodebuilder/init.go @@ -40,10 +40,10 @@ func Init(cfg Config, path string, tp node.Type) error { if err != nil { return fmt.Errorf("locking file: %w", err) } - defer flk.Close() if !ok { return ErrOpened } + defer flk.Unlock() //nolint:errcheck ksPath := keysPath(path) err = initDir(ksPath) @@ -88,10 +88,10 @@ func Reset(path string, tp node.Type) error { if err != nil { return fmt.Errorf("locking file: %w", err) } - defer flk.Close() if !ok { return ErrOpened } + defer flk.Unlock() //nolint:errcheck err = resetDir(dataPath(path)) if err != nil { diff --git a/nodebuilder/init_test.go b/nodebuilder/init_test.go index fffe2517e8..dcac1466af 100644 --- a/nodebuilder/init_test.go +++ b/nodebuilder/init_test.go @@ -61,7 +61,7 @@ func TestInitErrForLockedDir(t *testing.T) { flk := flock.New(lockPath(dir)) _, err := flk.TryLock() require.NoError(t, err) - defer flk.Close() + defer flk.Unlock() //nolint:errcheck nodes := []node.Type{node.Light, node.Bridge} for _, node := range nodes { diff --git a/nodebuilder/store.go b/nodebuilder/store.go index e5c051640c..ba6c19eaa2 100644 --- a/nodebuilder/store.go +++ b/nodebuilder/store.go @@ -68,13 +68,13 @@ func OpenStore(path string, ring keyring.Keyring) (Store, error) { } if !IsInit(path) { - err := errors.Join(ErrNotInited, flk.Close()) + err := errors.Join(ErrNotInited, flk.Unlock()) return nil, err } ks, err := keystore.NewFSKeystore(keysPath(path), ring) if err != nil { - err = errors.Join(err, flk.Close()) + err = errors.Join(err, flk.Unlock()) return nil, err } From 09629ebaf063faa37675bc721bc37633ad3e758c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:49:23 +0000 Subject: [PATCH 22/25] chore(deps): Bump mathieudutour/github-tag-action from 6.1 to 6.2 (#3252) Bumps [mathieudutour/github-tag-action](https://github.com/mathieudutour/github-tag-action) from 6.1 to 6.2.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=mathieudutour/github-tag-action&package-manager=github_actions&previous-version=6.1&new-version=6.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 3c17aca2e3..e914c967d0 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -98,7 +98,7 @@ jobs: # version_bump section, it would skip and not run, which would result # in goreleaser not running either. if: ${{ github.event_name == 'workflow_dispatch' }} - uses: mathieudutour/github-tag-action@v6.1 + uses: mathieudutour/github-tag-action@v6.2 with: github_token: ${{ secrets.GITHUB_TOKEN }} default_bump: ${{ inputs.version }} From c93713f69b937a3d8bbba221e1c8026512e6d190 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:19:35 +0000 Subject: [PATCH 23/25] chore(deps): Bump golang.org/x/crypto from 0.20.0 to 0.21.0 (#3249) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.20.0 to 0.21.0.
Commits
  • 7067223 go.mod: update golang.org/x dependencies
  • 0d2316b ssh/test: work around for TestCiphers failures on macOS
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/crypto&package-manager=go_modules&previous-version=0.20.0&new-version=0.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 913caa91df..564133553e 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( go.opentelemetry.io/proto/otlp v1.1.0 go.uber.org/fx v1.20.1 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.20.0 + golang.org/x/crypto v0.21.0 golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 @@ -322,8 +322,8 @@ require ( golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.16.1 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect diff --git a/go.sum b/go.sum index efaef1deeb..b062866923 100644 --- a/go.sum +++ b/go.sum @@ -2529,8 +2529,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= -golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2878,8 +2878,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2888,8 +2888,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 3549af881a5878ac07d442d60b7f110a859f3853 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Fri, 15 Mar 2024 22:56:02 +0400 Subject: [PATCH 24/25] feat(share/availability): persist random samples selection in availability call (#3239) This PR introduces persistence for sample selection in random sampling. It addresses the issue by storing all failed samples into the datastore, allowing them to be reloaded on the next sampling attempt. This ensures that if the availability call fails fully or partially during the last sampling attempt, the sampling retry will use the same preselected random coordinates of shares. Provided solution is backwards compatible with previously stored empty byte slice on sampling success, allowing the change to be non-breaking for existing storage. Additionally, this PR includes basic refactoring to simplify concurrency logic in availability. It also ensures that errors returned by the call are aligned with the interface declaration in [availability.go](https://github.com/celestiaorg/celestia-node/blob/main/share/availability.go) enhancing code consistency and maintainability. Resolves https://github.com/celestiaorg/celestia-node/issues/2780 --- share/availability/light/availability.go | 97 +++++++++++-------- share/availability/light/availability_test.go | 82 +++++++++------- share/availability/light/sample.go | 34 ++++++- share/availability/light/sample_test.go | 4 +- share/availability/light/testing.go | 47 +++++++++ 5 files changed, 182 insertions(+), 82 deletions(-) diff --git a/share/availability/light/availability.go b/share/availability/light/availability.go index 1d35542344..97046f4438 100644 --- a/share/availability/light/availability.go +++ b/share/availability/light/availability.go @@ -8,7 +8,6 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/autobatch" "github.com/ipfs/go-datastore/namespace" - ipldFormat "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" "github.com/celestiaorg/celestia-node/header" @@ -67,26 +66,37 @@ func (la *ShareAvailability) SharesAvailable(ctx context.Context, header *header return nil } - // do not sample over Root that has already been sampled + // load snapshot of the last sampling errors from disk key := rootKey(dah) - la.dsLk.RLock() - exists, err := la.ds.Has(ctx, key) + last, err := la.ds.Get(ctx, key) la.dsLk.RUnlock() - if err != nil || exists { + + // Check for error cases + var samples []Sample + switch { + case err == nil && len(last) == 0: + // Availability has already been validated + return nil + case err != nil && !errors.Is(err, datastore.ErrNotFound): + // Other error occurred return err + case errors.Is(err, datastore.ErrNotFound): + // No sampling result found, select new samples + samples, err = SampleSquare(len(dah.RowRoots), int(la.params.SampleAmount)) + if err != nil { + return err + } + default: + // Sampling result found, unmarshal it + samples, err = decodeSamples(last) + if err != nil { + return err + } } - log.Debugw("validate availability", "root", dah.String()) - // We assume the caller of this method has already performed basic validation on the - // given dah/root. If for some reason this has not happened, the node should panic. if err := dah.ValidateBasic(); err != nil { - log.Errorw("availability validation cannot be performed on a malformed DataAvailabilityHeader", - "err", err) - panic(err) - } - samples, err := SampleSquare(len(dah.RowRoots), int(la.params.SampleAmount)) - if err != nil { + log.Errorw("DAH validation failed", "error", err) return err } @@ -94,49 +104,50 @@ func (la *ShareAvailability) SharesAvailable(ctx context.Context, header *header // functionality is optional and must be supported by the used share.Getter. ctx = getters.WithSession(ctx) + var ( + failedSamplesLock sync.Mutex + failedSamples []Sample + ) + log.Debugw("starting sampling session", "root", dah.String()) - errs := make(chan error, len(samples)) + var wg sync.WaitGroup for _, s := range samples { + wg.Add(1) go func(s Sample) { - log.Debugw("fetching share", "root", dah.String(), "row", s.Row, "col", s.Col) - _, err := la.getter.GetShare(ctx, header, s.Row, s.Col) + defer wg.Done() + // check if the sample is available + _, err := la.getter.GetShare(ctx, header, int(s.Row), int(s.Col)) if err != nil { log.Debugw("error fetching share", "root", dah.String(), "row", s.Row, "col", s.Col) - } - // we don't really care about Share bodies at this point - // it also means we now saved the Share in local storage - select { - case errs <- err: - case <-ctx.Done(): + failedSamplesLock.Lock() + failedSamples = append(failedSamples, s) + failedSamplesLock.Unlock() } }(s) } + wg.Wait() - for range samples { - var err error - select { - case err = <-errs: - case <-ctx.Done(): - err = ctx.Err() - } - - if err != nil { - if errors.Is(err, context.Canceled) { - return err - } - log.Errorw("availability validation failed", "root", dah.String(), "err", err.Error()) - if ipldFormat.IsNotFound(err) || errors.Is(err, context.DeadlineExceeded) { - return share.ErrNotAvailable - } - return err - } + if errors.Is(ctx.Err(), context.Canceled) { + // Availability did not complete due to context cancellation, return context error instead of share.ErrNotAvailable + return ctx.Err() } + // store the result of the sampling session + bs := encodeSamples(failedSamples) la.dsLk.Lock() - err = la.ds.Put(ctx, key, []byte{}) + err = la.ds.Put(ctx, key, bs) la.dsLk.Unlock() if err != nil { - log.Errorw("storing root of successful SharesAvailable request to disk", "err", err) + log.Errorw("Failed to store sampling result", "error", err) + } + + // if any of the samples failed, return an error + if len(failedSamples) > 0 { + log.Errorw("availability validation failed", + "root", dah.String(), + "failed_samples", failedSamples, + ) + return share.ErrNotAvailable } return nil } diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index 2ace654d50..68da3698b5 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -6,7 +6,6 @@ import ( "strconv" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-node/header/headertest" @@ -26,16 +25,18 @@ func TestSharesAvailableCaches(t *testing.T) { // cache doesn't have dah yet has, err := avail.ds.Has(ctx, rootKey(dah)) - assert.NoError(t, err) - assert.False(t, has) + require.NoError(t, err) + require.False(t, has) err = avail.SharesAvailable(ctx, eh) - assert.NoError(t, err) + require.NoError(t, err) - // is now cached - has, err = avail.ds.Has(ctx, rootKey(dah)) - assert.NoError(t, err) - assert.True(t, has) + // is now stored success result + result, err := avail.ds.Get(ctx, rootKey(dah)) + require.NoError(t, err) + failed, err := decodeSamples(result) + require.NoError(t, err) + require.Empty(t, failed) } func TestSharesAvailableHitsCache(t *testing.T) { @@ -45,19 +46,16 @@ func TestSharesAvailableHitsCache(t *testing.T) { getter, _ := GetterWithRandSquare(t, 16) avail := TestAvailability(getter) + // create new dah, that is not available by getter bServ := ipld.NewMemBlockservice() dah := availability_test.RandFillBS(t, 16, bServ) eh := headertest.RandExtendedHeaderWithRoot(t, dah) // blockstore doesn't actually have the dah err := avail.SharesAvailable(ctx, eh) - require.Error(t, err) - - // cache doesn't have dah yet, since it errored - has, err := avail.ds.Has(ctx, rootKey(dah)) - assert.NoError(t, err) - assert.False(t, has) + require.ErrorIs(t, err, share.ErrNotAvailable) + // put success result in cache err = avail.ds.Put(ctx, rootKey(dah), []byte{}) require.NoError(t, err) @@ -75,31 +73,47 @@ func TestSharesAvailableEmptyRoot(t *testing.T) { eh := headertest.RandExtendedHeaderWithRoot(t, share.EmptyRoot()) err := avail.SharesAvailable(ctx, eh) - assert.NoError(t, err) + require.NoError(t, err) } -func TestSharesAvailable(t *testing.T) { +func TestSharesAvailableFailed(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - getter, dah := GetterWithRandSquare(t, 16) + getter, _ := GetterWithRandSquare(t, 16) avail := TestAvailability(getter) - err := avail.SharesAvailable(ctx, dah) - assert.NoError(t, err) -} - -func TestSharesAvailableFailed(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + // create new dah, that is not available by getter bServ := ipld.NewMemBlockservice() dah := availability_test.RandFillBS(t, 16, bServ) eh := headertest.RandExtendedHeaderWithRoot(t, dah) - getter, _ := GetterWithRandSquare(t, 16) - avail := TestAvailability(getter) + // blockstore doesn't actually have the dah, so it should fail err := avail.SharesAvailable(ctx, eh) - assert.Error(t, err) + require.ErrorIs(t, err, share.ErrNotAvailable) + + // cache should have failed results now + result, err := avail.ds.Get(ctx, rootKey(dah)) + require.NoError(t, err) + + failed, err := decodeSamples(result) + require.NoError(t, err) + require.Len(t, failed, int(avail.params.SampleAmount)) + + // ensure that retry persists the failed samples selection + // create new getter with only the failed samples available, and add them to the onceGetter + onceGetter := newOnceGetter() + onceGetter.AddSamples(failed) + + // replace getter with the new one + avail.getter = onceGetter + + // should be able to retrieve all the failed samples now + err = avail.SharesAvailable(ctx, eh) + require.NoError(t, err) + + // onceGetter should have no more samples stored after the call + require.Empty(t, onceGetter.available) } func TestShareAvailableOverMocknet_Light(t *testing.T) { @@ -114,7 +128,7 @@ func TestShareAvailableOverMocknet_Light(t *testing.T) { net.ConnectAll() err := nd.SharesAvailable(ctx, eh) - assert.NoError(t, err) + require.NoError(t, err) } func TestGetShare(t *testing.T) { @@ -127,8 +141,8 @@ func TestGetShare(t *testing.T) { for i := range make([]bool, n) { for j := range make([]bool, n) { sh, err := getter.GetShare(ctx, eh, i, j) - assert.NotNil(t, sh) - assert.NoError(t, err) + require.NotNil(t, sh) + require.NoError(t, err) } } } @@ -163,14 +177,14 @@ func TestService_GetSharesByNamespace(t *testing.T) { require.NoError(t, err) require.NoError(t, shares.Verify(root, randNamespace)) flattened := shares.Flatten() - assert.Len(t, flattened, tt.expectedShareCount) + require.Len(t, flattened, tt.expectedShareCount) for _, value := range flattened { - assert.Equal(t, randNamespace, share.GetNamespace(value)) + require.Equal(t, randNamespace, share.GetNamespace(value)) } if tt.expectedShareCount > 1 { // idx1 is always smaller than idx2 - assert.Equal(t, randShares[idx1], flattened[0]) - assert.Equal(t, randShares[idx2], flattened[1]) + require.Equal(t, randShares[idx1], flattened[0]) + require.Equal(t, randShares[idx2], flattened[1]) } }) t.Run("last two rows of a 4x4 square that have the same namespace have valid NMT proofs", func(t *testing.T) { diff --git a/share/availability/light/sample.go b/share/availability/light/sample.go index e66ff9aafe..e09a46a5fc 100644 --- a/share/availability/light/sample.go +++ b/share/availability/light/sample.go @@ -3,12 +3,14 @@ package light import ( crand "crypto/rand" + "encoding/binary" + "errors" "math/big" ) // Sample is a point in 2D space over square. type Sample struct { - Row, Col int + Row, Col uint16 } // SampleSquare randomly picks *num* unique points from the given *width* square @@ -66,11 +68,37 @@ func (ss *squareSampler) samples() []Sample { return samples } -func randInt(max int) int { +func randInt(max int) uint16 { n, err := crand.Int(crand.Reader, big.NewInt(int64(max))) if err != nil { panic(err) // won't panic as rand.Reader is endless } - return int(n.Int64()) + return uint16(n.Uint64()) +} + +// encodeSamples encodes a slice of samples into a byte slice using little endian encoding. +func encodeSamples(samples []Sample) []byte { + bs := make([]byte, 0, len(samples)*4) + for _, s := range samples { + bs = binary.LittleEndian.AppendUint16(bs, s.Row) + bs = binary.LittleEndian.AppendUint16(bs, s.Col) + } + return bs +} + +// decodeSamples decodes a byte slice into a slice of samples. +func decodeSamples(bs []byte) ([]Sample, error) { + if len(bs)%4 != 0 { + return nil, errors.New("invalid byte slice length") + } + + samples := make([]Sample, 0, len(bs)/4) + for i := 0; i < len(bs); i += 4 { + samples = append(samples, Sample{ + Row: binary.LittleEndian.Uint16(bs[i : i+2]), + Col: binary.LittleEndian.Uint16(bs[i+2 : i+4]), + }) + } + return samples, nil } diff --git a/share/availability/light/sample_test.go b/share/availability/light/sample_test.go index 7092b99e83..8d7656e688 100644 --- a/share/availability/light/sample_test.go +++ b/share/availability/light/sample_test.go @@ -21,8 +21,8 @@ func TestSampleSquare(t *testing.T) { assert.Len(t, ss, tt.samples) // check points are within width for _, s := range ss { - assert.Less(t, s.Row, tt.width) - assert.Less(t, s.Col, tt.width) + assert.Less(t, int(s.Row), tt.width) + assert.Less(t, int(s.Col), tt.width) } // checks samples are not equal for i, s1 := range ss { diff --git a/share/availability/light/testing.go b/share/availability/light/testing.go index 9efc9ff14a..b6251b4fbd 100644 --- a/share/availability/light/testing.go +++ b/share/availability/light/testing.go @@ -1,11 +1,15 @@ package light import ( + "context" + "sync" "testing" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-datastore" + "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" @@ -58,3 +62,46 @@ func SubNetNode(sn *availability_test.SubNet) *availability_test.TestNode { sn.AddNode(nd) return nd } + +type onceGetter struct { + *sync.Mutex + available map[Sample]struct{} +} + +func newOnceGetter() onceGetter { + return onceGetter{ + Mutex: &sync.Mutex{}, + available: make(map[Sample]struct{}), + } +} + +func (m onceGetter) AddSamples(samples []Sample) { + m.Lock() + defer m.Unlock() + for _, s := range samples { + m.available[s] = struct{}{} + } +} + +func (m onceGetter) GetShare(_ context.Context, _ *header.ExtendedHeader, row, col int) (share.Share, error) { + m.Lock() + defer m.Unlock() + s := Sample{Row: uint16(row), Col: uint16(col)} + if _, ok := m.available[s]; ok { + delete(m.available, s) + return share.Share{}, nil + } + return share.Share{}, share.ErrNotAvailable +} + +func (m onceGetter) GetEDS(_ context.Context, _ *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { + panic("not implemented") +} + +func (m onceGetter) GetSharesByNamespace( + _ context.Context, + _ *header.ExtendedHeader, + _ share.Namespace, +) (share.NamespacedShares, error) { + panic("not implemented") +} From 0c5ed2d7d7e929912b4f41b8598ef368b298352d Mon Sep 17 00:00:00 2001 From: smuu <18609909+smuu@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:36:48 +0100 Subject: [PATCH 25/25] feat: add workdir to dockerfile (#3259) This should be a non-breaking change. This only changes the work directory when the container is started. This means that if you want to execute a command in a running container, it will be automatically in that directory instead of '/'. This improves maintaining and debugging our environments. --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 66fbe6ae7a..5e29d82c43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,6 +59,8 @@ COPY --chown=${USER_NAME}:${USER_NAME} docker/entrypoint.sh /opt/entrypoint.sh USER ${USER_NAME} +WORKDIR ${CELESTIA_HOME} + EXPOSE 2121 ENTRYPOINT [ "/bin/bash", "/opt/entrypoint.sh" ]