Merge branch 'main' into chore/ramin/makefile-install

.github/workflows/ci_release.yml

@@ -4,9 +4,8 @@ on:
   push:
     branches:
       - main
-    # Trigger on version tags
-    tags:
-      - "v*"
+  release:
+    types: [published]
   pull_request:
   workflow_dispatch:
     inputs:
@@ -29,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       # upgrade go version throughout pipeline here
-      GO_VERSION: "1.21"
+      GO_VERSION: "1.22"
     outputs:
       go-version: ${{ steps.set-vars.outputs.go-version }}
       branch: ${{ steps.trim_ref.outputs.branch }}
@@ -111,7 +110,7 @@ jobs:
     runs-on: ubuntu-latest
     if: |
       github.event_name == 'workflow_dispatch' ||
-      (github.event_name == 'push' && contains(github.ref, 'refs/tags/'))
+      github.event_name == 'release'
     permissions: "write-all"
     steps:
       - uses: actions/checkout@v4

.github/workflows/github_pages.yml

@@ -0,0 +1,48 @@
+name: github-pages
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - specs/**
+  pull_request:
+    paths:
+      - specs/**
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup mdBook
+        uses: peaceiris/actions-mdbook@v1
+        with:
+          mdbook-version: "latest"
+
+      - name: Build book
+        run: mdbook build specs
+
+      - name: Deploy main
+        if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./specs/book
+          # keep_files is to prevent PR preview files from being overwritten.
+          # If we need to overwrite such files, trigger this workflow manually.
+          keep_files: ${{ github.event_name != 'workflow_dispatch' }}
+
+      - name: Deploy PR preview
+        # Only run this job if the PR was created from a branch on celestiaorg/celestia-node
+        # because this job will fail for branches from forks.
+        # https://github.com/celestiaorg/celestia-app/issues/1506
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
+        uses: rossjrw/pr-preview-action@v1
+        with:
+          source-dir: ./specs/book

.github/workflows/go-ci.yml

@@ -41,7 +41,7 @@ jobs:
           go-version: ${{ inputs.go-version }}
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3.7.0
+        uses: golangci/golangci-lint-action@v4.0.0
         with:
           args: --timeout 10m
           version: v1.55
@@ -98,7 +98,7 @@ jobs:
           retention-days: 5
 
       - name: upload coverage
-        uses: codecov/codecov-action@v3.1.6
+        uses: codecov/codecov-action@v4.1.0
         with:
           env_vars: OS
           token: ${{ secrets.CODECOV_TOKEN }}

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=$BUILDPLATFORM docker.io/golang:1.21-alpine3.18 as builder
+FROM --platform=$BUILDPLATFORM docker.io/golang:1.22-alpine3.18 as builder
 
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
@@ -40,16 +40,16 @@ ENV P2P_NETWORK mocha
 # hadolint ignore=DL3018
 RUN uname -a &&\
     apk update && apk add --no-cache \
-        bash \
-        curl \
-        jq \
+    bash \
+    curl \
+    jq \
     # Creates a user with $UID and $GID=$UID
     && adduser ${USER_NAME} \
-        -D \
-        -g ${USER_NAME} \
-        -h ${CELESTIA_HOME} \
-        -s /sbin/nologin \
-        -u ${UID}
+    -D \
+    -g ${USER_NAME} \
+    -h ${CELESTIA_HOME} \
+    -s /sbin/nologin \
+    -u ${UID}
 
 # Copy in the binary
 COPY --from=builder /src/build/celestia /bin/celestia

README.md

@@ -32,7 +32,7 @@ Continue reading [here](https://blog.celestia.org/celestia-mvp-release-data-avai
 
 | Requirement | Notes          |
 | ----------- |----------------|
-| Go version  | 1.21 or higher |
+| Go version  | 1.22 or higher |
 
 ## System Requirements
 

api/docgen/examples.go

@@ -3,6 +3,7 @@ package docgen
 import (
 	_ "embed"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"reflect"
 
@@ -66,7 +67,7 @@ var ExampleValues = map[reflect.Type]interface{}{
 			Shares: []*byzantine.ShareWithProof{},
 		},
 	),
-	reflect.TypeOf((*error)(nil)).Elem(): fmt.Errorf("error"),
+	reflect.TypeOf((*error)(nil)).Elem(): errors.New("error"),
 }
 
 func init() {

api/gateway/health.go

@@ -1,6 +1,8 @@
 package gateway
 
-import "net/http"
+import (
+	"net/http"
+)
 
 const (
 	healthEndpoint = "/status/health"

blob/blob.go

@@ -9,13 +9,14 @@ import (
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
 
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
-	"github.com/celestiaorg/celestia-app/pkg/shares"
 	"github.com/celestiaorg/celestia-app/x/blob/types"
 	"github.com/celestiaorg/nmt"
 
 	"github.com/celestiaorg/celestia-node/share"
 )
 
+var errEmptyShares = errors.New("empty shares")
+
 // Commitment is a Merkle Root of the subtree built from shares of the Blob.
 // It is computed by splitting the blob into shares and building the Merkle subtree to be included
 // after Submit.
@@ -31,37 +32,13 @@ func (com Commitment) Equal(c Commitment) bool {
 }
 
 // Proof is a collection of nmt.Proofs that verifies the inclusion of the data.
+// Proof proves the WHOLE namespaced data for the particular row.
+// TODO (@vgonkivs): rework `Proof` in order to prove a particular blob.
+// https://github.com/celestiaorg/celestia-node/issues/2303
 type Proof []*nmt.Proof
 
 func (p Proof) Len() int { return len(p) }
 
-func (p Proof) MarshalJSON() ([]byte, error) {
-	proofs := make([]string, 0, len(p))
-	for _, proof := range p {
-		proofBytes, err := proof.MarshalJSON()
-		if err != nil {
-			return nil, err
-		}
-		proofs = append(proofs, string(proofBytes))
-	}
-	return json.Marshal(proofs)
-}
-
-func (p *Proof) UnmarshalJSON(b []byte) error {
-	var proofs []string
-	if err := json.Unmarshal(b, &proofs); err != nil {
-		return err
-	}
-	for _, proof := range proofs {
-		var nmtProof nmt.Proof
-		if err := nmtProof.UnmarshalJSON([]byte(proof)); err != nil {
-			return err
-		}
-		*p = append(*p, &nmtProof)
-	}
-	return nil
-}
-
 // equal is a temporary method that compares two proofs.
 // should be removed in BlobService V1.
 func (p Proof) equal(input Proof) error {
@@ -99,6 +76,10 @@ type Blob struct {
 	// the celestia-node's namespace type
 	// this is to avoid converting to and from app's type
 	namespace share.Namespace
+
+	// index represents the index of the blob's first share in the EDS.
+	// Only retrieved, on-chain blobs will have the index set. Default is -1.
+	index int
 }
 
 // NewBlobV0 constructs a new blob from the provided Namespace and data.
@@ -127,19 +108,30 @@ func NewBlob(shareVersion uint8, namespace share.Namespace, data []byte) (*Blob,
 	if err != nil {
 		return nil, err
 	}
-	return &Blob{Blob: blob, Commitment: com, namespace: namespace}, nil
+	return &Blob{Blob: blob, Commitment: com, namespace: namespace, index: -1}, nil
 }
 
 // Namespace returns blob's namespace.
 func (b *Blob) Namespace() share.Namespace {
 	return b.namespace
 }
 
+// Index returns the blob's first share index in the EDS.
+// Only retrieved, on-chain blobs will have the index set. Default is -1.
+func (b *Blob) Index() int {
+	return b.index
+}
+
+func (b *Blob) compareCommitments(com Commitment) bool {
+	return bytes.Equal(b.Commitment, com)
+}
+
 type jsonBlob struct {
 	Namespace    share.Namespace `json:"namespace"`
 	Data         []byte          `json:"data"`
 	ShareVersion uint32          `json:"share_version"`
 	Commitment   Commitment      `json:"commitment"`
+	Index        int             `json:"index"`
 }
 
 func (b *Blob) MarshalJSON() ([]byte, error) {
@@ -148,6 +140,7 @@ func (b *Blob) MarshalJSON() ([]byte, error) {
 		Data:         b.Data,
 		ShareVersion: b.ShareVersion,
 		Commitment:   b.Commitment,
+		Index:        b.index,
 	}
 	return json.Marshal(blob)
 }
@@ -165,39 +158,6 @@ func (b *Blob) UnmarshalJSON(data []byte) error {
 	b.Blob.ShareVersion = blob.ShareVersion
 	b.Commitment = blob.Commitment
 	b.namespace = blob.Namespace
+	b.index = blob.Index
 	return nil
 }
-
-// buildBlobsIfExist takes shares and tries building the Blobs from them.
-// It will build blobs either until appShares will be empty or the first incomplete blob will
-// appear, so in this specific case it will return all built blobs + remaining shares.
-func buildBlobsIfExist(appShares []shares.Share) ([]*Blob, []shares.Share, error) {
-	if len(appShares) == 0 {
-		return nil, nil, errors.New("empty shares received")
-	}
-	blobs := make([]*Blob, 0, len(appShares))
-	for {
-		length, err := appShares[0].SequenceLen()
-		if err != nil {
-			return nil, nil, err
-		}
-
-		amount := shares.SparseSharesNeeded(length)
-		if amount > len(appShares) {
-			return blobs, appShares, nil
-		}
-
-		b, err := parseShares(appShares[:amount])
-		if err != nil {
-			return nil, nil, err
-		}
-
-		// only 1 blob will be created bc we passed the exact amount of shares
-		blobs = append(blobs, b[0])
-
-		if amount == len(appShares) {
-			return blobs, nil, nil
-		}
-		appShares = appShares[amount:]
-	}
-}

blob/blob_test.go

@@ -1,7 +1,7 @@
 package blob
 
 import (
-	"reflect"
+	"bytes"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -14,7 +14,7 @@ import (
 )
 
 func TestBlob(t *testing.T) {
-	appBlobs, err := blobtest.GenerateV0Blobs([]int{1}, false)
+	appBlobs, err := blobtest.GenerateV0Blobs([]int{16}, false)
 	require.NoError(t, err)
 	blob, err := convertBlobs(appBlobs...)
 	require.NoError(t, err)
@@ -53,10 +53,12 @@ func TestBlob(t *testing.T) {
 			expectedRes: func(t *testing.T) {
 				sh, err := BlobsToShares(blob...)
 				require.NoError(t, err)
-				b, err := SharesToBlobs(sh)
+				shares, err := toAppShares(sh...)
 				require.NoError(t, err)
-				assert.Equal(t, len(b), 1)
-				assert.Equal(t, blob[0].Commitment, b[0].Commitment)
+				p := &parser{length: len(shares), shares: shares}
+				b, err := p.parse()
+				require.NoError(t, err)
+				assert.Equal(t, blob[0].Commitment, b.Commitment)
 			},
 		},
 		{
@@ -67,7 +69,8 @@ func TestBlob(t *testing.T) {
 
 				newBlob := &Blob{}
 				require.NoError(t, newBlob.UnmarshalJSON(data))
-				require.True(t, reflect.DeepEqual(blob[0], newBlob))
+				require.True(t, bytes.Equal(blob[0].Blob.Data, newBlob.Data))
+				require.True(t, bytes.Equal(blob[0].Commitment, newBlob.Commitment))
 			},
 		},
 	}