Merge branch 'main' into feat/display-privatekey-discreetly

.github/workflows/go-ci.yml

@@ -98,28 +98,34 @@ jobs:
           retention-days: 5
 
       - name: upload coverage
-        uses: codecov/[email protected].4
+        uses: codecov/[email protected].6
         with:
           env_vars: OS
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ./coverage.txt
           name: coverage-${{ matrix.os }}
 
-  unit_test_race:
-    needs: [lint, go_mod_tidy_check]
-    name: Unit Tests with Race Detector (ubuntu-latest)
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: set up go
-        uses: actions/setup-go@v5
-        with:
-          go-version: ${{ inputs.go-version }}
-
-      - name: execute test run
-        run: make test-unit-race ENABLE_VERBOSE=${{ needs.setup.outputs.debug }}
+  # @ramin - Temporarily removed while we figure out getting
+  # these unit tests consistently running on ubuntu-latest
+  # and then enabled for macos-latest. We aren't requiring
+  # unit_race_test to pass for PRs so lets remove and reintroduce
+  # once green
+  #
+  # unit_test_race:
+  #   needs: [lint, go_mod_tidy_check]
+  #   name: Unit Tests with Race Detector (ubuntu-latest)
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - name: set up go
+  #       uses: actions/setup-go@v5
+  #       with:
+  #         go-version: ${{ inputs.go-version }}
+
+  #     - name: execute test run
+  #       run: make test-unit-race ENABLE_VERBOSE=${{ needs.setup.outputs.debug }}
 
   integration_test:
     name: Integration Tests

Dockerfile

@@ -25,7 +25,7 @@ RUN uname -a &&\
     CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
     make build && make cel-key
 
-FROM docker.io/alpine:3.19.0
+FROM docker.io/alpine:3.19.1
 
 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000
 ARG UID=10001

Makefile

@@ -166,7 +166,7 @@ pb-gen:
 ## openrpc-gen: Generate OpenRPC spec for Celestia-Node's RPC api
 openrpc-gen:
 	@echo "--> Generating OpenRPC spec"
-	@go run ./cmd/docgen fraud header state share das p2p node blob
+	@go run ./cmd/docgen fraud header state share das p2p node blob da
 .PHONY: openrpc-gen
 
 ## lint-imports: Lint only Go imports.

api/rpc/client/client.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/celestiaorg/celestia-node/api/rpc/perms"
 	"github.com/celestiaorg/celestia-node/nodebuilder/blob"
+	"github.com/celestiaorg/celestia-node/nodebuilder/da"
 	"github.com/celestiaorg/celestia-node/nodebuilder/das"
 	"github.com/celestiaorg/celestia-node/nodebuilder/fraud"
 	"github.com/celestiaorg/celestia-node/nodebuilder/header"
@@ -33,6 +34,7 @@ type Client struct {
 	P2P    p2p.API
 	Node   node.API
 	Blob   blob.API
+	DA     da.API
 
 	closer multiClientCloser
 }
@@ -91,5 +93,6 @@ func moduleMap(client *Client) map[string]interface{} {
 		"p2p":    &client.P2P.Internal,
 		"node":   &client.Node.Internal,
 		"blob":   &client.Blob.Internal,
+		"da":     &client.DA.Internal,
 	}
 }

api/rpc/server.go

@@ -20,16 +20,17 @@ import (
 var log = logging.Logger("rpc")
 
 type Server struct {
-	srv      *http.Server
-	rpc      *jsonrpc.RPCServer
-	listener net.Listener
+	srv          *http.Server
+	rpc          *jsonrpc.RPCServer
+	listener     net.Listener
+	authDisabled bool
 
 	started atomic.Bool
 
 	auth jwt.Signer
 }
 
-func NewServer(address, port string, secret jwt.Signer) *Server {
+func NewServer(address, port string, authDisabled bool, secret jwt.Signer) *Server {
 	rpc := jsonrpc.NewServer()
 	srv := &Server{
 		rpc: rpc,
@@ -38,7 +39,8 @@ func NewServer(address, port string, secret jwt.Signer) *Server {
 			// the amount of time allowed to read request headers. set to the default 2 seconds
 			ReadHeaderTimeout: 2 * time.Second,
 		},
-		auth: secret,
+		auth:         secret,
+		authDisabled: authDisabled,
 	}
 	srv.srv.Handler = &auth.Handler{
 		Verify: srv.verifyAuth,
@@ -51,6 +53,9 @@ func NewServer(address, port string, secret jwt.Signer) *Server {
 // reached if a token is provided in the header of the request, otherwise only
 // methods with `read` permissions are accessible.
 func (s *Server) verifyAuth(_ context.Context, token string) ([]auth.Permission, error) {
+	if s.authDisabled {
+		return perms.AllPerms, nil
+	}
 	return authtoken.ExtractSignedPermissions(s.auth, token)
 }
 

api/rpc_test.go

@@ -22,6 +22,8 @@ import (
 	"github.com/celestiaorg/celestia-node/nodebuilder"
 	"github.com/celestiaorg/celestia-node/nodebuilder/blob"
 	blobMock "github.com/celestiaorg/celestia-node/nodebuilder/blob/mocks"
+	"github.com/celestiaorg/celestia-node/nodebuilder/da"
+	daMock "github.com/celestiaorg/celestia-node/nodebuilder/da/mocks"
 	"github.com/celestiaorg/celestia-node/nodebuilder/das"
 	dasMock "github.com/celestiaorg/celestia-node/nodebuilder/das/mocks"
 	"github.com/celestiaorg/celestia-node/nodebuilder/fraud"
@@ -91,6 +93,7 @@ type api struct {
 	Node   node.Module
 	P2P    p2p.Module
 	Blob   blob.Module
+	DA     da.Module
 }
 
 func TestModulesImplementFullAPI(t *testing.T) {
@@ -297,6 +300,7 @@ func setupNodeWithAuthedRPC(t *testing.T, auth jwt.Signer) (*nodebuilder.Node, *
 		p2pMock.NewMockModule(ctrl),
 		nodeMock.NewMockModule(ctrl),
 		blobMock.NewMockModule(ctrl),
+		daMock.NewMockModule(ctrl),
 	}
 
 	// given the behavior of fx.Invoke, this invoke will be called last as it is added at the root
@@ -310,6 +314,7 @@ func setupNodeWithAuthedRPC(t *testing.T, auth jwt.Signer) (*nodebuilder.Node, *
 		srv.RegisterAuthedService("p2p", mockAPI.P2P, &p2p.API{})
 		srv.RegisterAuthedService("node", mockAPI.Node, &node.API{})
 		srv.RegisterAuthedService("blob", mockAPI.Blob, &blob.API{})
+		srv.RegisterAuthedService("da", mockAPI.DA, &da.API{})
 	})
 	// fx.Replace does not work here, but fx.Decorate does
 	nd := nodebuilder.TestNode(t, node.Full, invokeRPC, fx.Decorate(func() (jwt.Signer, error) {
@@ -334,4 +339,5 @@ type mockAPI struct {
 	P2P    *p2pMock.MockModule
 	Node   *nodeMock.MockModule
 	Blob   *blobMock.MockModule
+	DA     *daMock.MockModule
 }

blob/blob.go

@@ -35,6 +35,33 @@ type Proof []*nmt.Proof
 
 func (p Proof) Len() int { return len(p) }
 
+func (p Proof) MarshalJSON() ([]byte, error) {
+	proofs := make([]string, 0, len(p))
+	for _, proof := range p {
+		proofBytes, err := proof.MarshalJSON()
+		if err != nil {
+			return nil, err
+		}
+		proofs = append(proofs, string(proofBytes))
+	}
+	return json.Marshal(proofs)
+}
+
+func (p *Proof) UnmarshalJSON(b []byte) error {
+	var proofs []string
+	if err := json.Unmarshal(b, &proofs); err != nil {
+		return err
+	}
+	for _, proof := range proofs {
+		var nmtProof nmt.Proof
+		if err := nmtProof.UnmarshalJSON([]byte(proof)); err != nil {
+			return err
+		}
+		*p = append(*p, &nmtProof)
+	}
+	return nil
+}
+
 // equal is a temporary method that compares two proofs.
 // should be removed in BlobService V1.
 func (p Proof) equal(input Proof) error {

blob/service.go

@@ -11,20 +11,26 @@ import (
 	"github.com/cosmos/cosmos-sdk/types"
 	auth "github.com/cosmos/cosmos-sdk/x/auth/types"
 	logging "github.com/ipfs/go-log/v2"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
 	"github.com/celestiaorg/celestia-app/pkg/shares"
 	blobtypes "github.com/celestiaorg/celestia-app/x/blob/types"
 
 	"github.com/celestiaorg/celestia-node/header"
+	"github.com/celestiaorg/celestia-node/libs/utils"
 	"github.com/celestiaorg/celestia-node/share"
 )
 
 var (
 	ErrBlobNotFound = errors.New("blob: not found")
 	ErrInvalidProof = errors.New("blob: invalid proof")
 
-	log = logging.Logger("blob")
+	log    = logging.Logger("blob")
+	tracer = otel.Tracer("blob/service")
 )
 
 // GasPrice represents the amount to be paid per gas unit. Fee is set by
@@ -185,7 +191,11 @@ func (s *Service) Included(
 	namespace share.Namespace,
 	proof *Proof,
 	com Commitment,
-) (bool, error) {
+) (_ bool, err error) {
+	ctx, span := tracer.Start(ctx, "included")
+	defer func() {
+		utils.SetStatusAndEnd(span, err)
+	}()
 	// In the current implementation, LNs will have to download all shares to recompute the commitment.
 	// To achieve 1. we need to modify Proof structure and to store all subtree roots, that were
 	// involved in commitment creation and then call `merkle.HashFromByteSlices`(tendermint package).
@@ -213,24 +223,47 @@ func (s *Service) getByCommitment(
 	height uint64,
 	namespace share.Namespace,
 	commitment Commitment,
-) (*Blob, *Proof, error) {
+) (_ *Blob, _ *Proof, err error) {
 	log.Infow("requesting blob",
 		"height", height,
 		"namespace", namespace.String())
 
-	header, err := s.headerGetter(ctx, height)
+	ctx, span := tracer.Start(ctx, "get-by-commitment")
+	defer func() {
+		utils.SetStatusAndEnd(span, err)
+	}()
+	span.SetAttributes(
+		attribute.Int64("height", int64(height)),
+		attribute.String("commitment", string(commitment)),
+	)
+
+	getCtx, headerGetterSpan := tracer.Start(ctx, "header-getter")
+
+	header, err := s.headerGetter(getCtx, height)
 	if err != nil {
+		headerGetterSpan.SetStatus(codes.Error, err.Error())
 		return nil, nil, err
 	}
 
-	namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header, namespace)
+	headerGetterSpan.SetStatus(codes.Ok, "")
+	headerGetterSpan.AddEvent("received eds", trace.WithAttributes(
+		attribute.Int64("eds-size", int64(len(header.DAH.RowRoots)))))
+
+	getCtx, getSharesSpan := tracer.Start(ctx, "get-shares-by-namespace")
+
+	namespacedShares, err := s.shareGetter.GetSharesByNamespace(getCtx, header, namespace)
 	if err != nil {
 		if errors.Is(err, share.ErrNotFound) {
 			err = ErrBlobNotFound
 		}
+		getSharesSpan.SetStatus(codes.Error, err.Error())
 		return nil, nil, err
 	}
 
+	getSharesSpan.SetStatus(codes.Ok, "")
+	getSharesSpan.AddEvent("received shares", trace.WithAttributes(
+		attribute.Int64("eds-size", int64(len(header.DAH.RowRoots)))))
+
 	var (
 		rawShares = make([]shares.Share, 0)
 		proofs    = make(Proof, 0)
@@ -260,6 +293,7 @@ func (s *Service) getByCommitment(
 		}
 		for _, b := range blobs {
 			if b.Commitment.Equal(commitment) {
+				span.AddEvent("blob reconstructed")
 				return b, &proofs, nil
 			}
 			// Falling under this flag means that the data from the last row
@@ -296,7 +330,11 @@ func (s *Service) getBlobs(
 	ctx context.Context,
 	namespace share.Namespace,
 	header *header.ExtendedHeader,
-) ([]*Blob, error) {
+) (_ []*Blob, err error) {
+	ctx, span := tracer.Start(ctx, "get-blobs")
+	defer func() {
+		utils.SetStatusAndEnd(span, err)
+	}()
 	namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header, namespace)
 	if err != nil {
 		return nil, err

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/benbjohnson/clock v1.3.5
 	github.com/celestiaorg/celestia-app v1.4.0
 	github.com/celestiaorg/go-fraud v0.2.0
-	github.com/celestiaorg/go-header v0.5.2
+	github.com/celestiaorg/go-header v0.5.3
 	github.com/celestiaorg/go-libp2p-messenger v0.2.0
 	github.com/celestiaorg/nmt v0.20.0
 	github.com/celestiaorg/rsmt2d v0.11.0
@@ -46,13 +46,14 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/muesli/termenv v0.15.2
 	github.com/multiformats/go-base32 v0.1.0
-	github.com/multiformats/go-multiaddr v0.12.1
+	github.com/multiformats/go-multiaddr v0.12.2
 	github.com/multiformats/go-multiaddr-dns v0.3.1
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333
 	github.com/prometheus/client_golang v1.18.0
 	github.com/pyroscope-io/client v0.7.2
 	github.com/pyroscope-io/otel-profiling-go v0.5.0
+	github.com/rollkit/go-da v0.4.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4

go.sum

@@ -370,8 +370,8 @@ github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJ
 github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI=
 github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I=
 github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc=
-github.com/celestiaorg/go-header v0.5.2 h1:CFsTAXcs1o38JVd8YN1Naq/Yzs6m9orMPEPNpLEgFJA=
-github.com/celestiaorg/go-header v0.5.2/go.mod h1:7BVR6myjRfACbqW1de6s8OjuK66XzHm8MpFNYr0G+nU=
+github.com/celestiaorg/go-header v0.5.3 h1:8CcflT6aIlcQXKNWcMekoBNs3EU50mEmDp17gbn1pP4=
+github.com/celestiaorg/go-header v0.5.3/go.mod h1:7BVR6myjRfACbqW1de6s8OjuK66XzHm8MpFNYr0G+nU=
 github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao=
 github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo=
 github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc=
@@ -1837,8 +1837,8 @@ github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar
 github.com/multiformats/go-multiaddr v0.6.0/go.mod h1:F4IpaKZuPP360tOMn2Tpyu0At8w23aRyVqeK0DbFeGM=
 github.com/multiformats/go-multiaddr v0.7.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs=
 github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs=
-github.com/multiformats/go-multiaddr v0.12.1 h1:vm+BA/WZA8QZDp1pF1FWhi5CT3g1tbi5GJmqpb6wnlk=
-github.com/multiformats/go-multiaddr v0.12.1/go.mod h1:7mPkiBMmLeFipt+nNSq9pHZUeJSt8lHBgH6yhj0YQzE=
+github.com/multiformats/go-multiaddr v0.12.2 h1:9G9sTY/wCYajKa9lyfWPmpZAwe6oV+Wb1zcmMS1HG24=
+github.com/multiformats/go-multiaddr v0.12.2/go.mod h1:GKyaTYjZRdcUhyOetrxTk9z0cW+jA/YrnqTOvKgi44M=
 github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0=
@@ -2128,6 +2128,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/rollkit/go-da v0.4.0 h1:/s7ZrVq7DC2aK8UXIvB7rsXrZ2mVGRw7zrexcxRvhlw=
+github.com/rollkit/go-da v0.4.0/go.mod h1:Kef0XI5ecEKd3TXzI8S+9knAUJnZg0svh2DuXoCsPlM=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U=
 github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=

libs/pidstore/pidstore_test.go

@@ -22,7 +22,7 @@ func TestPutLoad(t *testing.T) {
 
 	ds := sync.MutexWrap(datastore.NewMapDatastore())
 
-	t.Run("unitialized-pidstore", func(t *testing.T) {
+	t.Run("uninitialized-pidstore", func(t *testing.T) {
 		testPutLoad(ctx, ds, t)
 	})
 	t.Run("initialized-pidstore", func(t *testing.T) {