From f89b1d91f3a5cfbba1f1a71862efe656fb543c1b Mon Sep 17 00:00:00 2001 From: Javed Khan Date: Wed, 5 Jul 2023 13:08:44 -0700 Subject: [PATCH] state: update SubmitPayForBlob api; api: add BlobAPI (#11) ## Overview This PR adds `type Blob`, adds `BlobAPI` and updates the API for `SubmitPayForBlob` to the latest version of celestia-node. Fixes #10 ## Checklist - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [x] Required CI checks are passing - [x] Visual proof for any user facing features like CLI or documentation updates - [x] Linked issues closed with keywords --- api.go | 15 ++- client.go | 7 +- types/appconsts/global_consts.go | 78 +++++++++++++ types/appconsts/initial_consts.go | 29 +++++ types/blob/blob.go | 145 ++++++++++++++++++++++++ types/core/core.go | 56 ++++++++- types/header/header.go | 60 +++++++++- types/namespace/consts.go | 75 ++++++++++++ types/namespace/namespace.go | 157 ++++++++++++++++++++++++- types/namespace/random_namespace.go | 18 +++ types/share/errors.go | 8 ++ types/share/namespace.go | 170 ++++++++++++++++++++++++++++ types/share/share.go | 43 ++++++- 13 files changed, 849 insertions(+), 12 deletions(-) create mode 100644 types/appconsts/global_consts.go create mode 100644 types/appconsts/initial_consts.go create mode 100644 types/blob/blob.go create mode 100644 types/namespace/consts.go create mode 100644 types/namespace/random_namespace.go create mode 100644 types/share/errors.go create mode 100644 types/share/namespace.go diff --git a/api.go b/api.go index 08a9a31..56f6fe6 100644 --- a/api.go +++ b/api.go @@ -14,9 +14,9 @@ import ( "github.com/libp2p/go-libp2p/core/protocol" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" + "github.com/rollkit/celestia-openrpc/types/blob" "github.com/rollkit/celestia-openrpc/types/das" "github.com/rollkit/celestia-openrpc/types/header" - "github.com/rollkit/celestia-openrpc/types/namespace" "github.com/rollkit/celestia-openrpc/types/node" "github.com/rollkit/celestia-openrpc/types/share" "github.com/rollkit/celestia-openrpc/types/state" @@ -36,6 +36,14 @@ type DASAPI struct { WaitCatchUp func(ctx context.Context) error `perm:"read"` } +type BlobAPI struct { + Submit func(context.Context, []*blob.Blob) (uint64, error) `perm:"write"` + Get func(context.Context, uint64, share.Namespace, blob.Commitment) (*blob.Blob, error) `perm:"read"` + GetAll func(context.Context, uint64, []share.Namespace) ([]*blob.Blob, error) `perm:"read"` + GetProof func(context.Context, uint64, share.Namespace, blob.Commitment) (*blob.Proof, error) `perm:"read"` + Included func(context.Context, uint64, share.Namespace, *blob.Proof, blob.Commitment) (bool, error) `perm:"read"` +} + type HeaderAPI struct { LocalHead func(context.Context) (*header.ExtendedHeader, error) `perm:"read"` GetByHash func( @@ -68,10 +76,9 @@ type StateAPI struct { SubmitTx func(ctx context.Context, tx state.Tx) (*state.TxResponse, error) `perm:"write"` SubmitPayForBlob func( ctx context.Context, - nID namespace.ID, - data []byte, fee state.Int, gasLim uint64, + blobs []*blob.Blob, ) (*state.TxResponse, error) `perm:"write"` CancelUnbondingDelegation func( ctx context.Context, @@ -132,7 +139,7 @@ type ShareAPI struct { GetSharesByNamespace func( ctx context.Context, root *share.Root, - namespace namespace.ID, + namespace share.Namespace, ) (share.NamespacedShares, error) `perm:"public"` } type P2PAPI struct { diff --git a/client.go b/client.go index c417a70..0950670 100644 --- a/client.go +++ b/client.go @@ -12,6 +12,7 @@ const AuthKey = "Authorization" type Client struct { Fraud FraudAPI + Blob BlobAPI Header HeaderAPI State StateAPI Share ShareAPI @@ -45,12 +46,16 @@ func (c *Client) Close() { } func NewClient(ctx context.Context, addr string, token string) (*Client, error) { - authHeader := http.Header{AuthKey: []string{fmt.Sprintf("Bearer %s", token)}} + var authHeader http.Header + if token != "" { + authHeader = http.Header{AuthKey: []string{fmt.Sprintf("Bearer %s", token)}} + } var client Client modules := map[string]interface{}{ "fraud": &client.Fraud, + "blob": &client.Blob, "header": &client.Header, "state": &client.State, "share": &client.Share, diff --git a/types/appconsts/global_consts.go b/types/appconsts/global_consts.go new file mode 100644 index 0000000..1d6c9f8 --- /dev/null +++ b/types/appconsts/global_consts.go @@ -0,0 +1,78 @@ +package appconsts + +import ( + "math" + + "github.com/celestiaorg/rsmt2d" +) + +// These constants were originally sourced from: +// https://github.com/celestiaorg/celestia-specs/blob/master/src/specs/consensus.md#constants +// +// They can not change throughout the lifetime of a network. +const ( + // NamespaceVersionSize is the size of a namespace version in bytes. + NamespaceVersionSize = 1 + // NamespaceVersionMaxValue is the maximum value a namespace version can be. + // This const must be updated if NamespaceVersionSize is changed. + NamespaceVersionMaxValue = math.MaxUint8 + + // NamespaceIDSize is the size of a namespace ID in bytes. + NamespaceIDSize = 28 + + // NamespaceSize is the size of a namespace (version + ID) in bytes. + NamespaceSize = NamespaceVersionSize + NamespaceIDSize + + // ShareSize is the size of a share in bytes. + ShareSize = 512 + + // ShareInfoBytes is the number of bytes reserved for information. The info + // byte contains the share version and a sequence start idicator. + ShareInfoBytes = 1 + + // SequenceLenBytes is the number of bytes reserved for the sequence length + // that is present in the first share of a sequence. + SequenceLenBytes = 4 + + // ShareVersionZero is the first share version format. + ShareVersionZero = uint8(0) + + // DefaultShareVersion is the defacto share version. Use this if you are + // unsure of which version to use. + DefaultShareVersion = ShareVersionZero + + // CompactShareReservedBytes is the number of bytes reserved for the location of + // the first unit (transaction, ISR) in a compact share. + CompactShareReservedBytes = 4 + + // FirstCompactShareContentSize is the number of bytes usable for data in + // the first compact share of a sequence. + FirstCompactShareContentSize = ShareSize - NamespaceSize - ShareInfoBytes - SequenceLenBytes - CompactShareReservedBytes + + // ContinuationCompactShareContentSize is the number of bytes usable for + // data in a continuation compact share of a sequence. + ContinuationCompactShareContentSize = ShareSize - NamespaceSize - ShareInfoBytes - CompactShareReservedBytes + + // FirstSparseShareContentSize is the number of bytes usable for data in the + // first sparse share of a sequence. + FirstSparseShareContentSize = ShareSize - NamespaceSize - ShareInfoBytes - SequenceLenBytes + + // ContinuationSparseShareContentSize is the number of bytes usable for data + // in a continuation sparse share of a sequence. + ContinuationSparseShareContentSize = ShareSize - NamespaceSize - ShareInfoBytes + + // MinSquareSize is the smallest original square width. + MinSquareSize = 1 + + // MinshareCount is the minimum number of shares allowed in the original + // data square. + MinShareCount = MinSquareSize * MinSquareSize + + // MaxShareVersion is the maximum value a share version can be. + MaxShareVersion = 127 +) + +var ( + // DefaultCodec is the default codec creator used for data erasure. + DefaultCodec = rsmt2d.NewLeoRSCodec +) diff --git a/types/appconsts/initial_consts.go b/types/appconsts/initial_consts.go new file mode 100644 index 0000000..7f98e74 --- /dev/null +++ b/types/appconsts/initial_consts.go @@ -0,0 +1,29 @@ +package appconsts + +import "time" + +// The following defaults correspond to initial parameters of the network that can be changed, not via app versions +// but other means such as on-chain governance, or the nodes local config +const ( + // DefaultGovMaxSquareSize is the default value for the governance modifiable + // max square size. + DefaultGovMaxSquareSize = 64 + + // DefaultMaxBytes is the default value for the governance modifiable + // maximum number of bytes allowed in a valid block. + DefaultMaxBytes = DefaultGovMaxSquareSize * DefaultGovMaxSquareSize * ContinuationSparseShareContentSize + + // DefaultGasPerBlobByte is the default gas cost deducted per byte of blob + // included in a PayForBlobs txn + DefaultGasPerBlobByte = 8 + + // DefaultMinGasPrice is the default min gas price that gets set in the app.toml file. + // The min gas price acts as a filter. Transactions below that limit will not pass + // a nodes `CheckTx` and thus not be proposed by that node. + DefaultMinGasPrice = 0.1 + + // DefaultUnbondingTime is the default time a validator must wait + // to unbond in a proof of stake system. Any validator within this + // time can be subject to slashing under conditions of misbehavior. + DefaultUnbondingTime = 3 * 7 * 24 * time.Hour +) diff --git a/types/blob/blob.go b/types/blob/blob.go new file mode 100644 index 0000000..5dae022 --- /dev/null +++ b/types/blob/blob.go @@ -0,0 +1,145 @@ +package blob + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + + "github.com/celestiaorg/nmt" + + "github.com/rollkit/celestia-openrpc/types/appconsts" + "github.com/rollkit/celestia-openrpc/types/share" +) + +const ( + // NMTIgnoreMaxNamespace is currently used value for IgnoreMaxNamespace option in NMT. + // IgnoreMaxNamespace defines whether the largest possible Namespace MAX_NID should be 'ignored'. + // If set to true, this allows for shorter proofs in particular use-cases. + NMTIgnoreMaxNamespace = true +) + +var ( + ErrBlobNotFound = errors.New("blob: not found") + ErrInvalidProof = errors.New("blob: invalid proof") +) + +// Commitment is a Merkle Root of the subtree built from shares of the Blob. +// It is computed by splitting the blob into shares and building the Merkle subtree to be included +// after Submit. +type Commitment []byte + +func (com Commitment) String() string { + return string(com) +} + +// Equal ensures that commitments are the same +func (com Commitment) Equal(c Commitment) bool { + return bytes.Equal(com, c) +} + +// Proof is a collection of nmt.Proofs that verifies the inclusion of the data. +type Proof []*nmt.Proof + +func (p Proof) Len() int { return len(p) } + +type jsonProof struct { + Start int `json:"start"` + End int `json:"end"` + Nodes [][]byte `json:"nodes"` +} + +func (p *Proof) MarshalJSON() ([]byte, error) { + proofs := make([]jsonProof, 0, p.Len()) + for _, pp := range *p { + proofs = append(proofs, jsonProof{ + Start: pp.Start(), + End: pp.End(), + Nodes: pp.Nodes(), + }) + } + + return json.Marshal(proofs) +} + +func (p *Proof) UnmarshalJSON(data []byte) error { + var proofs []jsonProof + err := json.Unmarshal(data, &proofs) + if err != nil { + return err + } + + nmtProofs := make([]*nmt.Proof, len(proofs)) + for i, jProof := range proofs { + nmtProof := nmt.NewInclusionProof(jProof.Start, jProof.End, jProof.Nodes, NMTIgnoreMaxNamespace) + nmtProofs[i] = &nmtProof + } + + *p = nmtProofs + return nil +} + +// Blob represents any application-specific binary data that anyone can submit to Celestia. +type Blob struct { + Namespace []byte `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + ShareVersion uint32 `protobuf:"varint,3,opt,name=share_version,json=shareVersion,proto3" json:"share_version,omitempty"` + NamespaceVersion uint32 `protobuf:"varint,4,opt,name=namespace_version,json=namespaceVersion,proto3" json:"namespace_version,omitempty"` + Commitment []byte `protobuf:"bytes,5,opt,name=commitment,proto3" json:"commitment,omitempty"` +} + +// NewBlobV0 constructs a new blob from the provided Namespace and data. +// The blob will be formatted as v0 shares. +func NewBlobV0(namespace share.Namespace, data []byte) (*Blob, error) { + return NewBlob(appconsts.ShareVersionZero, namespace, data) +} + +// NewBlob constructs a new blob from the provided Namespace, data and share version. +func NewBlob(shareVersion uint8, namespace share.Namespace, data []byte) (*Blob, error) { + if len(data) == 0 || len(data) > appconsts.DefaultMaxBytes { + return nil, fmt.Errorf("blob data must be > 0 && <= %d, but it was %d bytes", appconsts.DefaultMaxBytes, len(data)) + } + if err := namespace.ValidateForBlob(); err != nil { + return nil, err + } + + return &Blob{ + Namespace: namespace, + Data: data, + ShareVersion: uint32(shareVersion), + NamespaceVersion: 0, + Commitment: []byte{}, + }, nil +} + +type jsonBlob struct { + Namespace share.Namespace `json:"namespace"` + Data []byte `json:"data"` + ShareVersion uint32 `json:"share_version"` + Commitment Commitment `json:"commitment"` +} + +func (b *Blob) MarshalJSON() ([]byte, error) { + blob := &jsonBlob{ + Namespace: b.Namespace, + Data: b.Data, + ShareVersion: b.ShareVersion, + Commitment: b.Commitment, + } + return json.Marshal(blob) +} + +func (b *Blob) UnmarshalJSON(data []byte) error { + var blob jsonBlob + err := json.Unmarshal(data, &blob) + if err != nil { + return err + } + + b.NamespaceVersion = uint32(blob.Namespace.Version()) + b.Data = blob.Data + b.ShareVersion = blob.ShareVersion + b.Commitment = blob.Commitment + b.Namespace = blob.Namespace + return nil +} diff --git a/types/core/core.go b/types/core/core.go index b9f52c0..f3be21a 100644 --- a/types/core/core.go +++ b/types/core/core.go @@ -1,9 +1,13 @@ package core import ( + "bytes" + "fmt" "time" + "github.com/celestiaorg/rsmt2d" "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/merkle" tmbytes "github.com/tendermint/tendermint/libs/bytes" tmversion "github.com/tendermint/tendermint/proto/tendermint/version" ) @@ -17,7 +21,7 @@ type Header struct { // basic block info Version tmversion.Consensus `json:"version"` ChainID string `json:"chain_id"` - Height int64 `json:"height"` + Height int64 `json:"height,string"` Time time.Time `json:"time"` // prev block info @@ -118,9 +122,57 @@ type Validator struct { // https://github.com/celestiaorg/celestia-specs/blob/master/src/specs/data_structures.md#availabledataheader type DataAvailabilityHeader struct { // RowRoot_j = root((M_{j,1} || M_{j,2} || ... || M_{j,2k} )) - RowsRoots [][]byte `json:"row_roots"` + RowRoots [][]byte `json:"row_roots"` // ColumnRoot_j = root((M_{1,j} || M_{2,j} || ... || M_{2k,j} )) ColumnRoots [][]byte `json:"column_roots"` // hash is the Merkle root of the row and column roots. This field is the // memoized result from `Hash()`. + hash []byte +} + +// NewDataAvailabilityHeader generates a DataAvailability header using the provided square size and shares +func NewDataAvailabilityHeader(eds *rsmt2d.ExtendedDataSquare) DataAvailabilityHeader { + // generate the row and col roots using the EDS + dah := DataAvailabilityHeader{ + RowRoots: eds.RowRoots(), + ColumnRoots: eds.ColRoots(), + } + + // generate the hash of the data using the new roots + dah.Hash() + + return dah +} + +// String returns hex representation of merkle hash of the DAHeader. +func (dah *DataAvailabilityHeader) String() string { + if dah == nil { + return "" + } + return fmt.Sprintf("%X", dah.Hash()) +} + +// Equals checks equality of two DAHeaders. +func (dah *DataAvailabilityHeader) Equals(to *DataAvailabilityHeader) bool { + return bytes.Equal(dah.Hash(), to.Hash()) +} + +// Hash computes the Merkle root of the row and column roots. Hash memoizes the +// result in `DataAvailabilityHeader.hash`. +func (dah *DataAvailabilityHeader) Hash() []byte { + if dah == nil { + return merkle.HashFromByteSlices(nil) + } + if len(dah.hash) != 0 { + return dah.hash + } + + rowsCount := len(dah.RowRoots) + slices := make([][]byte, rowsCount+rowsCount) + copy(slices[0:rowsCount], dah.RowRoots) + copy(slices[rowsCount:], dah.ColumnRoots) + // The single data root is computed using a simple binary merkle tree. + // Effectively being root(rowRoots || columnRoots): + dah.hash = merkle.HashFromByteSlices(slices) + return dah.hash } diff --git a/types/header/header.go b/types/header/header.go index 86278e1..9626ffe 100644 --- a/types/header/header.go +++ b/types/header/header.go @@ -1,6 +1,12 @@ package header -import "github.com/rollkit/celestia-openrpc/types/core" +import ( + "encoding/json" + + tmjson "github.com/tendermint/tendermint/libs/json" + + "github.com/rollkit/celestia-openrpc/types/core" +) // RawHeader is an alias to core.Header. It is // "raw" because it is not yet wrapped to include @@ -17,4 +23,56 @@ type ExtendedHeader struct { DAH *DataAvailabilityHeader `json:"dah"` } +// MarshalJSON marshals an ExtendedHeader to JSON. The ValidatorSet is wrapped with amino encoding, +// to be able to unmarshal the crypto.PubKey type back from JSON. +func (eh *ExtendedHeader) MarshalJSON() ([]byte, error) { + type Alias ExtendedHeader + validatorSet, err := tmjson.Marshal(eh.ValidatorSet) + if err != nil { + return nil, err + } + rawHeader, err := tmjson.Marshal(eh.RawHeader) + if err != nil { + return nil, err + } + return json.Marshal(&struct { + RawHeader json.RawMessage `json:"header"` + ValidatorSet json.RawMessage `json:"validator_set"` + *Alias + }{ + ValidatorSet: validatorSet, + RawHeader: rawHeader, + Alias: (*Alias)(eh), + }) +} + +// UnmarshalJSON unmarshals an ExtendedHeader from JSON. The ValidatorSet is wrapped with amino +// encoding, to be able to unmarshal the crypto.PubKey type back from JSON. +func (eh *ExtendedHeader) UnmarshalJSON(data []byte) error { + type Alias ExtendedHeader + aux := &struct { + RawHeader json.RawMessage `json:"header"` + ValidatorSet json.RawMessage `json:"validator_set"` + *Alias + }{ + Alias: (*Alias)(eh), + } + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + valSet := new(core.ValidatorSet) + if err := tmjson.Unmarshal(aux.ValidatorSet, valSet); err != nil { + return err + } + rawHeader := new(RawHeader) + if err := tmjson.Unmarshal(aux.RawHeader, rawHeader); err != nil { + return err + } + + eh.ValidatorSet = valSet + eh.RawHeader = *rawHeader + return nil +} + type DataAvailabilityHeader = core.DataAvailabilityHeader diff --git a/types/namespace/consts.go b/types/namespace/consts.go new file mode 100644 index 0000000..9316a5c --- /dev/null +++ b/types/namespace/consts.go @@ -0,0 +1,75 @@ +package namespace + +import ( + "bytes" + "crypto/sha256" + "math" +) + +const ( + // NamespaveVersionSize is the size of a namespace version in bytes. + NamespaceVersionSize = 1 + + // NamespaceIDSize is the size of a namespace ID in bytes. + NamespaceIDSize = 28 + + // NamespaceSize is the size of a namespace (version + ID) in bytes. + NamespaceSize = NamespaceVersionSize + NamespaceIDSize + + // NamespaceVersionZero is the first namespace version. + NamespaceVersionZero = uint8(0) + + // NamespaceVersionMax is the max namespace version. + NamespaceVersionMax = math.MaxUint8 + + // NamespaceZeroPrefixSize is the number of `0` bytes that are prefixed to + // namespace IDs for version 0. + NamespaceVersionZeroPrefixSize = 18 + + // NamespaceVersionZeroIDSize is the number of bytes available for + // user-specified namespace ID in a namespace ID for version 0. + NamespaceVersionZeroIDSize = NamespaceIDSize - NamespaceVersionZeroPrefixSize +) + +var ( + // NamespaceVersionZeroPrefix is the prefix of a namespace ID for version 0. + NamespaceVersionZeroPrefix = bytes.Repeat([]byte{0}, NamespaceVersionZeroPrefixSize) + + // TxNamespace is the namespace reserved for transaction data. + TxNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) + + // IntermediateStateRootsNamespace is the namespace reserved for + // intermediate state root data. + IntermediateStateRootsNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) + + // PayForBlobNamespace is the namespace reserved for PayForBlobs transactions. + PayForBlobNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 4}) + + // ReservedPaddingNamespace is the namespace used for padding after all + // reserved namespaces. In practice this padding is after transactions + // (ordinary and PFBs) but before blobs. + ReservedPaddingNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) + + // MaxReservedNamespace is lexicographically the largest namespace that is + // reserved for protocol use. + MaxReservedNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) + + // TailPaddingNamespace is the namespace reserved for tail padding. All data + // with this namespace will be ignored. + TailPaddingNamespace = Namespace{ + Version: math.MaxUint8, + ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 0xFE), + } + + // ParitySharesNamespace is the namespace reserved for erasure coded data. + ParitySharesNamespace = Namespace{ + Version: math.MaxUint8, + ID: bytes.Repeat([]byte{0xFF}, NamespaceIDSize), + } +) + +var ( + // NewBaseHashFunc is the base hash function used by NMT. Change accordingly + // if another hash.Hash should be used as a base hasher in the NMT. + NewBaseHashFunc = sha256.New +) diff --git a/types/namespace/namespace.go b/types/namespace/namespace.go index 3abe39b..427b062 100644 --- a/types/namespace/namespace.go +++ b/types/namespace/namespace.go @@ -1,3 +1,158 @@ package namespace -type ID []byte +import ( + "bytes" + "fmt" +) + +type Namespace struct { + Version uint8 + ID []byte +} + +// New returns a new namespace with the provided version and id. +func New(version uint8, id []byte) (Namespace, error) { + err := validateVersion(version) + if err != nil { + return Namespace{}, err + } + + err = validateID(version, id) + if err != nil { + return Namespace{}, err + } + + return Namespace{ + Version: version, + ID: id, + }, nil +} + +// MustNew returns a new namespace with the provided version and id. It panics +// if the provided version or id are not supported. +func MustNew(version uint8, id []byte) Namespace { + ns, err := New(version, id) + if err != nil { + panic(err) + } + return ns +} + +// MustNewV0 returns a new namespace with version 0 and the provided id. This +// function panics if the provided id is not exactly NamespaceVersionZeroIDSize bytes. +func MustNewV0(id []byte) Namespace { + if len(id) != NamespaceVersionZeroIDSize { + panic(fmt.Sprintf("invalid namespace id length: %v must be %v", len(id), NamespaceVersionZeroIDSize)) + } + + ns, err := New(NamespaceVersionZero, append(NamespaceVersionZeroPrefix, id...)) + if err != nil { + panic(err) + } + return ns +} + +// From returns a namespace from the provided byte slice. +func From(b []byte) (Namespace, error) { + if len(b) != NamespaceSize { + return Namespace{}, fmt.Errorf("invalid namespace length: %v must be %v", len(b), NamespaceSize) + } + rawVersion := b[0] + rawNamespace := b[1:] + return New(rawVersion, rawNamespace) +} + +// Bytes returns this namespace as a byte slice. +func (n Namespace) Bytes() []byte { + return append([]byte{n.Version}, n.ID...) +} + +// ValidateBlobNamespace returns an error if this namespace is not a valid blob namespace. +func (n Namespace) ValidateBlobNamespace() error { + if n.IsReserved() { + return fmt.Errorf("invalid blob namespace: %v cannot use a reserved namespace ID, want > %v", n.Bytes(), MaxReservedNamespace.Bytes()) + } + + if n.IsParityShares() { + return fmt.Errorf("invalid blob namespace: %v cannot use parity shares namespace ID", n.Bytes()) + } + + if n.IsTailPadding() { + return fmt.Errorf("invalid blob namespace: %v cannot use tail padding namespace ID", n.Bytes()) + } + + return nil +} + +// validateVersion returns an error if the version is not supported. +func validateVersion(version uint8) error { + if version != NamespaceVersionZero && version != NamespaceVersionMax { + return fmt.Errorf("unsupported namespace version %v", version) + } + return nil +} + +// validateID returns an error if the provided id does not meet the requirements +// for the provided version. +func validateID(version uint8, id []byte) error { + if len(id) != NamespaceIDSize { + return fmt.Errorf("unsupported namespace id length: id %v must be %v bytes but it was %v bytes", id, NamespaceIDSize, len(id)) + } + + if version == NamespaceVersionZero && !bytes.HasPrefix(id, NamespaceVersionZeroPrefix) { + return fmt.Errorf("unsupported namespace id with version %v. ID %v must start with %v leading zeros", version, id, len(NamespaceVersionZeroPrefix)) + } + return nil +} + +func (n Namespace) IsReserved() bool { + return bytes.Compare(n.Bytes(), MaxReservedNamespace.Bytes()) < 1 +} + +func (n Namespace) IsParityShares() bool { + return bytes.Equal(n.Bytes(), ParitySharesNamespace.Bytes()) +} + +func (n Namespace) IsTailPadding() bool { + return bytes.Equal(n.Bytes(), TailPaddingNamespace.Bytes()) +} + +func (n Namespace) IsReservedPadding() bool { + return bytes.Equal(n.Bytes(), ReservedPaddingNamespace.Bytes()) +} + +func (n Namespace) IsTx() bool { + return bytes.Equal(n.Bytes(), TxNamespace.Bytes()) +} + +func (n Namespace) IsPayForBlob() bool { + return bytes.Equal(n.Bytes(), PayForBlobNamespace.Bytes()) +} + +func (n Namespace) Repeat(times int) []Namespace { + ns := make([]Namespace, times) + for i := 0; i < times; i++ { + ns[i] = n + } + return ns +} + +func (n Namespace) Equals(n2 Namespace) bool { + return bytes.Equal(n.Bytes(), n2.Bytes()) +} + +func (n Namespace) IsLessThan(n2 Namespace) bool { + return bytes.Compare(n.Bytes(), n2.Bytes()) == -1 +} + +func (n Namespace) IsLessOrEqualThan(n2 Namespace) bool { + return bytes.Compare(n.Bytes(), n2.Bytes()) < 1 +} + +func (n Namespace) IsGreaterThan(n2 Namespace) bool { + return bytes.Compare(n.Bytes(), n2.Bytes()) == 1 +} + +func (n Namespace) IsGreaterOrEqualThan(n2 Namespace) bool { + return bytes.Compare(n.Bytes(), n2.Bytes()) > -1 +} diff --git a/types/namespace/random_namespace.go b/types/namespace/random_namespace.go new file mode 100644 index 0000000..79e52a6 --- /dev/null +++ b/types/namespace/random_namespace.go @@ -0,0 +1,18 @@ +package namespace + +import tmrand "github.com/tendermint/tendermint/libs/rand" + +func RandomNamespace() Namespace { + for { + id := RandomVerzionZeroID() + namespace, err := New(NamespaceVersionZero, id) + if err != nil { + continue + } + return namespace + } +} + +func RandomVerzionZeroID() []byte { + return append(NamespaceVersionZeroPrefix, tmrand.Bytes(NamespaceVersionZeroIDSize)...) +} diff --git a/types/share/errors.go b/types/share/errors.go new file mode 100644 index 0000000..6af00a9 --- /dev/null +++ b/types/share/errors.go @@ -0,0 +1,8 @@ +package share + +import "errors" + +var ( + // ErrNotAvailable is returned whenever DA sampling fails. + ErrNotAvailable = errors.New("share: data not available") +) diff --git a/types/share/namespace.go b/types/share/namespace.go new file mode 100644 index 0000000..d9a674e --- /dev/null +++ b/types/share/namespace.go @@ -0,0 +1,170 @@ +package share + +import ( + "bytes" + "encoding/hex" + "fmt" + + "github.com/celestiaorg/nmt/namespace" + + "github.com/rollkit/celestia-openrpc/types/appconsts" + appns "github.com/rollkit/celestia-openrpc/types/namespace" +) + +// Various reserved namespaces. +var ( + MaxReservedNamespace = Namespace(appns.MaxReservedNamespace.Bytes()) + ParitySharesNamespace = Namespace(appns.ParitySharesNamespace.Bytes()) + TailPaddingNamespace = Namespace(appns.TailPaddingNamespace.Bytes()) + ReservedPaddingNamespace = Namespace(appns.ReservedPaddingNamespace.Bytes()) + TxNamespace = Namespace(appns.TxNamespace.Bytes()) + PayForBlobNamespace = Namespace(appns.PayForBlobNamespace.Bytes()) +) + +// Namespace represents namespace of a Share. +// Consists of version byte and namespace ID. +type Namespace []byte + +// NewBlobNamespaceV0 takes a variable size byte slice and creates a valid version 0 Blob Namespace. +// The byte slice must be <= 10 bytes. +// If it is less than 10 bytes, it will be left padded to size 10 with 0s. +// Use predefined namespaces above, if non-blob namespace is needed. +func NewBlobNamespaceV0(id []byte) (Namespace, error) { + if len(id) == 0 || len(id) > appns.NamespaceVersionZeroIDSize { + return nil, fmt.Errorf( + "namespace id must be > 0 && <= %d, but it was %d bytes", appns.NamespaceVersionZeroIDSize, len(id)) + } + + n := make(Namespace, appconsts.NamespaceSize) + // version and zero padding are already set as zero, + // so simply copying subNID to the end is enough to comply the V0 spec + copy(n[len(n)-len(id):], id) + return n, n.ValidateForBlob() +} + +// NamespaceFromBytes converts bytes into Namespace and validates it. +func NamespaceFromBytes(b []byte) (Namespace, error) { + n := Namespace(b) + return n, n.Validate() +} + +// Version reports version of the Namespace. +func (n Namespace) Version() byte { + return n[appns.NamespaceVersionSize-1] +} + +// ID reports ID of the Namespace. +func (n Namespace) ID() namespace.ID { + return namespace.ID(n[appns.NamespaceVersionSize:]) +} + +// ToNMT converts the whole Namespace(both Version and ID parts) into NMT's namespace.ID +// NOTE: Once https://github.com/celestiaorg/nmt/issues/206 is closed Namespace should become NNT's +// type. +func (n Namespace) ToNMT() namespace.ID { + return namespace.ID(n) +} + +// ToAppNamespace converts the Namespace to App's definition of Namespace. +// TODO: Unify types between node and app +func (n Namespace) ToAppNamespace() appns.Namespace { + return appns.Namespace{Version: n.Version(), ID: n.ID()} +} + +// Len reports the total length of the namespace. +func (n Namespace) Len() int { + return len(n) +} + +// String stringifies the Namespace. +func (n Namespace) String() string { + return hex.EncodeToString(n) +} + +// Equals compares two Namespaces. +func (n Namespace) Equals(target Namespace) bool { + return bytes.Equal(n, target) +} + +// Validate checks if the namespace is correct. +func (n Namespace) Validate() error { + if n.Len() != appconsts.NamespaceSize { + return fmt.Errorf("invalid namespace length: expected %d, got %d", appconsts.NamespaceSize, n.Len()) + } + if n.Version() != appns.NamespaceVersionZero && n.Version() != appns.NamespaceVersionMax { + return fmt.Errorf("invalid namespace version %v", n.Version()) + } + if len(n.ID()) != appns.NamespaceIDSize { + return fmt.Errorf("invalid namespace id length: expected %d, got %d", appns.NamespaceIDSize, n.ID().Size()) + } + if n.Version() == appns.NamespaceVersionZero && !bytes.HasPrefix(n.ID(), appns.NamespaceVersionZeroPrefix) { + return fmt.Errorf("invalid namespace id: expect %d leading zeroes", len(appns.NamespaceVersionZeroPrefix)) + } + return nil +} + +// ValidateForData checks if the Namespace is of real/useful data. +func (n Namespace) ValidateForData() error { + if err := n.Validate(); err != nil { + return err + } + if n.Equals(ParitySharesNamespace) || n.Equals(TailPaddingNamespace) { + return fmt.Errorf("invalid data namespace(%s): parity and tail padding namespace are forbidden", n) + } + return nil +} + +// ValidateForBlob checks if the Namespace is valid blob namespace. +func (n Namespace) ValidateForBlob() error { + if err := n.ValidateForData(); err != nil { + return err + } + if bytes.Compare(n, MaxReservedNamespace) < 1 { + return fmt.Errorf("invalid blob namespace(%s): reserved namespaces are forbidden", n) + } + return nil +} + +// IsAboveMax checks if the namespace is above the maximum namespace of the given hash. +func (n Namespace) IsAboveMax(nodeHash []byte) bool { + return !n.IsLessOrEqual(nodeHash[n.Len() : n.Len()*2]) +} + +// IsBelowMin checks if the target namespace is below the minimum namespace of the given hash. +func (n Namespace) IsBelowMin(nodeHash []byte) bool { + return n.IsLess(nodeHash[:n.Len()]) +} + +// IsOutsideRange checks if the namespace is outside the min-max range of the given hashes. +func (n Namespace) IsOutsideRange(leftNodeHash, rightNodeHash []byte) bool { + return n.IsBelowMin(leftNodeHash) || n.IsAboveMax(rightNodeHash) +} + +// Repeat copies the Namespace t times. +func (n Namespace) Repeat(t int) []Namespace { + ns := make([]Namespace, t) + for i := 0; i < t; i++ { + ns[i] = n + } + return ns +} + +// IsLess reports if the Namespace is less than the target. +func (n Namespace) IsLess(target Namespace) bool { + return bytes.Compare(n, target) == -1 +} + +// IsLessOrEqual reports if the Namespace is less than the target. +func (n Namespace) IsLessOrEqual(target Namespace) bool { + return bytes.Compare(n, target) < 1 +} + +// IsGreater reports if the Namespace is greater than the target. +func (n Namespace) IsGreater(target Namespace) bool { + return bytes.Compare(n, target) == 1 +} + +// IsGreaterOrEqualThan reports if the Namespace is greater or equal than the target. +func (n Namespace) IsGreaterOrEqualThan(target Namespace) bool { + return bytes.Compare(n, target) > -1 +} diff --git a/types/share/share.go b/types/share/share.go index 21d1e30..d4574ad 100644 --- a/types/share/share.go +++ b/types/share/share.go @@ -1,8 +1,11 @@ package share import ( + "fmt" + "github.com/celestiaorg/nmt" + "github.com/rollkit/celestia-openrpc/types/appconsts" "github.com/rollkit/celestia-openrpc/types/core" ) @@ -10,17 +13,51 @@ import ( // In practice, it is a commitment to all the Data in a square. type Root = core.DataAvailabilityHeader -// NamespacedShares represents all shares with proofs within a specific namespace of an EDS. -type NamespacedShares []NamespacedRow - // NamespacedRow represents all shares with proofs within a specific namespace of a single EDS row. type NamespacedRow struct { Shares []Share Proof *nmt.Proof } +// NamespacedShares represents all shares with proofs within a specific namespace of an EDS. +type NamespacedShares []NamespacedRow + +var ( + // DefaultRSMT2DCodec sets the default rsmt2d.Codec for shares. + DefaultRSMT2DCodec = appconsts.DefaultCodec +) + +const ( + // Size is a system-wide size of a share, including both data and namespace GetNamespace + Size = appconsts.ShareSize +) + // Share contains the raw share data without the corresponding namespace. // NOTE: Alias for the byte is chosen to keep maximal compatibility, especially with rsmt2d. // Ideally, we should define reusable type elsewhere and make everyone(Core, rsmt2d, ipld) to rely // on it. type Share = []byte + +// GetNamespace slices Namespace out of the Share. +func GetNamespace(s Share) Namespace { + return s[:appconsts.NamespaceSize] +} + +// GetData slices out data of the Share. +func GetData(s Share) []byte { + return s[appconsts.NamespaceSize:] +} + +// DataHash is a representation of the Root hash. +type DataHash []byte + +func (dh DataHash) Validate() error { + if len(dh) != 32 { + return fmt.Errorf("invalid hash size, expected 32, got %d", len(dh)) + } + return nil +} + +func (dh DataHash) String() string { + return fmt.Sprintf("%X", []byte(dh)) +}