Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

core(da): da path and metadata refactor #1355

Merged
merged 47 commits into from
Feb 4, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
8fa2289
openrpc removed
srene Jan 17, 2025
95ad1e1
new headers api
srene Jan 17, 2025
b3ee48d
getheaders fix + balance
srene Jan 18, 2025
dba4923
mock da
srene Jan 20, 2025
7cb7495
missing file
srene Jan 20, 2025
0791a1d
fixing tests
srene Jan 20, 2025
7182308
fix tests
srene Jan 21, 2025
ecde45c
namespace
srene Jan 21, 2025
0dd5dc2
lint fix
srene Jan 21, 2025
749dcc5
Merge branch 'main' into srene/celestia-refactor
srene Jan 21, 2025
71ee67e
missing file
srene Jan 21, 2025
430bedc
error added
srene Jan 21, 2025
731c5dc
retrieval err handling
srene Jan 23, 2025
8d8ca0e
remove da syncing wait
srene Jan 23, 2025
45aa3a7
lint fix
srene Jan 23, 2025
0c1902d
godoc
srene Jan 23, 2025
86e4e6f
submit celestia refactor
srene Jan 23, 2025
c92f891
cleaning
srene Jan 23, 2025
254ce7d
error
srene Jan 23, 2025
9863444
remove go-da dep
srene Jan 23, 2025
5ecfc67
rm wait for syncing
srene Jan 23, 2025
64925dd
Merge branch 'main' into srene/celestia-refactor
srene Jan 23, 2025
81db522
go doc
srene Jan 23, 2025
be628b6
fix test
srene Jan 23, 2025
3319bc3
lint
srene Jan 23, 2025
9a4e1e3
dapath refactor
srene Jan 29, 2025
c96345f
grpc
srene Jan 30, 2025
f1260d2
weavm path
srene Jan 30, 2025
3c4436b
celestia test ok
srene Jan 30, 2025
ce53172
manager test fix
srene Jan 30, 2025
0bdfa32
da test fix
srene Jan 30, 2025
dec623a
weavevm test fix
srene Jan 30, 2025
99a52a2
dym test fix
srene Jan 30, 2025
9a94232
da validator test fix
srene Jan 30, 2025
2df6c64
lint fix
srene Jan 30, 2025
b329f76
minor
srene Jan 30, 2025
c03aa12
clean
srene Jan 30, 2025
84927ca
go doc
srene Jan 30, 2025
c17797d
missing files
srene Jan 31, 2025
e3f2638
improve fraud detection
srene Jan 31, 2025
779d716
da type avail fix
srene Jan 31, 2025
101e0d8
da paths parts check
srene Feb 1, 2025
5e78be4
merge
srene Feb 1, 2025
b08fe4b
remove check metadata struct
srene Feb 4, 2025
4c5751d
merge
srene Feb 4, 2025
e29f6ec
Update block/retriever.go
srene Feb 4, 2025
7e44c6d
Update da/avail/avail.go
srene Feb 4, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions block/fraud_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -318,9 +318,7 @@ func TestApplyBatchFromSLWithFraud(t *testing.T) {
})
}
slBatch := &settlement.Batch{
MetaData: &settlement.BatchMetaData{
DA: daResultSubmitBatch.SubmitMetaData,
},
MetaData: daResultSubmitBatch.SubmitMetaData,
BlockDescriptors: bds,
}

Expand Down
18 changes: 9 additions & 9 deletions block/manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/libp2p/go-libp2p/core/crypto"

"github.com/dymensionxyz/dymint/block"
"github.com/dymensionxyz/dymint/da/local"
"github.com/dymensionxyz/dymint/p2p"
"github.com/dymensionxyz/dymint/settlement"
"github.com/dymensionxyz/dymint/testutil"
Expand Down Expand Up @@ -207,12 +208,13 @@ func TestRetrieveDaBatchesFailed(t *testing.T) {
manager.DAClient = testutil.GetMockDALC(log.TestingLogger())
manager.Retriever = manager.DAClient.(da.BatchRetriever)

submitMetadata := local.SubmitMetaData{
Height: 1,
}
batch := &settlement.Batch{
MetaData: &settlement.BatchMetaData{
DA: &da.DASubmitMetaData{
Client: da.Mock,
Height: 1,
},
MetaData: &da.DASubmitMetaData{
Client: da.Mock,
DAPath: submitMetadata.ToPath(),
},
}

Expand Down Expand Up @@ -545,7 +547,7 @@ func TestDAFetch(t *testing.T) {
{
name: "wrong DA",
manager: manager,
daMetaData: &da.DASubmitMetaData{Client: da.Celestia, Height: daResultSubmitBatch.SubmitMetaData.Height},
daMetaData: &da.DASubmitMetaData{Client: da.Celestia, DAPath: daResultSubmitBatch.SubmitMetaData.ToPath()},
batch: batch,
err: da.ErrDAMismatch,
},
Expand All @@ -566,9 +568,7 @@ func TestDAFetch(t *testing.T) {
})
}
slBatch := &settlement.Batch{
MetaData: &settlement.BatchMetaData{
DA: c.daMetaData,
},
MetaData: c.daMetaData,
BlockDescriptors: bds,
EndHeight: batch.EndHeight(),
}
Expand Down
10 changes: 5 additions & 5 deletions block/retriever.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,13 @@ import (
)

func (m *Manager) ApplyBatchFromSL(slBatch *settlement.Batch) error {
m.logger.Debug("trying to retrieve batch from DA", "daHeight", slBatch.MetaData.DA.Height)
batchResp := m.fetchBatch(slBatch.MetaData.DA)
m.logger.Debug("trying to retrieve batch from DA", "DA Metadata", slBatch.MetaData)
batchResp := m.fetchBatch(slBatch.MetaData)
if batchResp.Code != da.StatusSuccess {
return batchResp.Error
}

m.logger.Debug("retrieved batches", "n", len(batchResp.Batches), "daHeight", slBatch.MetaData.DA.Height)
m.logger.Debug("retrieved batches", "n", len(batchResp.Batches), "dametadata", slBatch.MetaData)

m.retrieverMu.Lock()
defer m.retrieverMu.Unlock()
Expand Down Expand Up @@ -46,7 +46,7 @@ func (m *Manager) ApplyBatchFromSL(slBatch *settlement.Batch) error {
}

// We dont validate because validateBlockBeforeApply already checks if the block is already applied, and we don't need to fail there.
err := m.validateAndApplyBlock(block, batch.Commits[i], types.BlockMetaData{Source: types.DA, DAHeight: slBatch.MetaData.DA.Height})
err := m.validateAndApplyBlock(block, batch.Commits[i], types.BlockMetaData{Source: types.DA})
if err != nil {
return fmt.Errorf("apply block: height: %d: %w", block.Header.Height, err)
}
Expand Down Expand Up @@ -113,7 +113,7 @@ func (m *Manager) fetchBatch(daMetaData *da.DASubmitMetaData) da.ResultRetrieveB
}

// batchRes.MetaData includes proofs necessary to open disputes with the Hub
batchRes := m.Retriever.RetrieveBatches(daMetaData)
batchRes := m.Retriever.RetrieveBatches(daMetaData.DAPath)
// TODO(srene) : for invalid transactions there is no specific error code since it will need to be validated somewhere else for fraud proving.
// NMT proofs (availRes.MetaData.Proofs) are included in the result batchRes, necessary to be included in the dispute
return batchRes
Expand Down
9 changes: 4 additions & 5 deletions block/slvalidator.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package block
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"sync/atomic"
Expand Down Expand Up @@ -73,20 +72,20 @@ func (v *SettlementValidator) ValidateStateUpdate(batch *settlement.ResultRetrie
// load all DA blocks from the batch to be validated
var daBatch da.ResultRetrieveBatch
for {
daBatch = v.blockManager.Retriever.RetrieveBatches(batch.MetaData.DA)
daBatch = v.blockManager.fetchBatch(batch.MetaData)
if daBatch.Code == da.StatusSuccess {
break
}

// fraud detected in case blob is retrieved but unable to get blocks from it.
if errors.Is(daBatch.BaseResult.Error, da.ErrBlobNotParsed) {
return types.NewErrStateUpdateBlobCorruptedFraud(batch.StateIndex, string(batch.MetaData.DA.Client), batch.MetaData.DA.Height, hex.EncodeToString(batch.MetaData.DA.Commitment))
return types.NewErrStateUpdateBlobCorruptedFraud(batch.StateIndex, string(batch.MetaData.Client), batch.MetaData.DAPath)
}

// fraud detected in case availability checks fail and therefore there certainty the blob, according to the state update DA path, is not available.
checkBatchResult := v.blockManager.Retriever.CheckBatchAvailability(batch.MetaData.DA)
checkBatchResult := v.blockManager.Retriever.CheckBatchAvailability(batch.MetaData.DAPath)
if errors.Is(checkBatchResult.Error, da.ErrBlobNotIncluded) {
return types.NewErrStateUpdateBlobNotAvailableFraud(batch.StateIndex, string(batch.MetaData.DA.Client), batch.MetaData.DA.Height, hex.EncodeToString(batch.MetaData.DA.Commitment))
return types.NewErrStateUpdateBlobNotAvailableFraud(batch.StateIndex, string(batch.MetaData.Client), batch.MetaData.DAPath)
}

// FIXME: how to handle non-happy case? not returning error?
Expand Down
12 changes: 5 additions & 7 deletions block/slvalidator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -414,13 +414,11 @@ func getSLBatch(bds []rollapp.BlockDescriptor, daMetaData *da.DASubmitMetaData,
return &settlement.ResultRetrieveBatch{
Batch: &settlement.Batch{
BlockDescriptors: bds,
MetaData: &settlement.BatchMetaData{
DA: daMetaData,
},
StartHeight: startHeight,
EndHeight: endHeight,
NumBlocks: endHeight - startHeight + 1,
NextSequencer: nextSequencer,
MetaData: daMetaData,
StartHeight: startHeight,
EndHeight: endHeight,
NumBlocks: endHeight - startHeight + 1,
NextSequencer: nextSequencer,
},
ResultBase: settlement.ResultBase{
StateIndex: 1,
Expand Down
41 changes: 35 additions & 6 deletions da/avail/avail.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"math"
"strconv"
"time"

"github.com/avast/retry-go/v4"
Expand Down Expand Up @@ -71,6 +72,31 @@ type DataAvailabilityLayerClient struct {
synced chan struct{}
}

// SubmitMetaData contains meta data about a batch on the Data Availability Layer.
type SubmitMetaData struct {
// Height is the height of the block in the da layer
Height uint64
}

// ToPath converts a SubmitMetaData to a path.
func (d *SubmitMetaData) ToPath() string {
return strconv.FormatUint(d.Height, 10)
}

// FromPath parses a path to a SubmitMetaData.
func (d *SubmitMetaData) FromPath(path string) (*SubmitMetaData, error) {
height, err := strconv.ParseUint(path, 10, 64)
if err != nil {
return nil, err
}

submitData := &SubmitMetaData{
Height: height,
}

return submitData, nil
}

var (
_ da.DataAvailabilityLayerClient = &DataAvailabilityLayerClient{}
_ da.BatchRetriever = &DataAvailabilityLayerClient{}
Expand Down Expand Up @@ -169,7 +195,12 @@ func (c *DataAvailabilityLayerClient) GetClientType() da.Client {
}

// RetrieveBatches retrieves batch from DataAvailabilityLayerClient instance.
func (c *DataAvailabilityLayerClient) RetrieveBatches(daMetaData *da.DASubmitMetaData) da.ResultRetrieveBatch {
func (c *DataAvailabilityLayerClient) RetrieveBatches(daPath string) da.ResultRetrieveBatch {
daMetaData := &SubmitMetaData{}
daMetaData, err := daMetaData.FromPath(daPath)
if err != nil {
return da.ResultRetrieveBatch{BaseResult: da.BaseResult{Code: da.StatusError, Message: "read da path", Error: err}}
}
//nolint:typecheck
blockHash, err := c.client.GetBlockHash(daMetaData.Height)
if err != nil {
Expand Down Expand Up @@ -238,9 +269,6 @@ func (c *DataAvailabilityLayerClient) RetrieveBatches(daMetaData *da.DASubmitMet
BaseResult: da.BaseResult{
Code: da.StatusSuccess,
},
CheckMetaData: &da.DACheckMetaData{
Height: daMetaData.Height,
},
Batches: batches,
}
}
Expand Down Expand Up @@ -308,6 +336,7 @@ func (c *DataAvailabilityLayerClient) submitBatchLoop(dataBlob []byte) da.Result
continue
}
metrics.RollappConsecutiveFailedDASubmission.Set(0)
submitMetadata := &SubmitMetaData{Height: daBlockHeight}

c.logger.Debug("Successfully submitted batch.")
return da.ResultSubmitBatch{
Expand All @@ -316,8 +345,8 @@ func (c *DataAvailabilityLayerClient) submitBatchLoop(dataBlob []byte) da.Result
Message: "success",
},
SubmitMetaData: &da.DASubmitMetaData{
DAPath: submitMetadata.ToPath(),
Client: da.Avail,
Height: daBlockHeight,
},
}
}
Expand Down Expand Up @@ -425,7 +454,7 @@ func (c *DataAvailabilityLayerClient) broadcastTx(tx []byte) (uint64, error) {
}

// CheckBatchAvailability checks batch availability in DataAvailabilityLayerClient instance.
func (c *DataAvailabilityLayerClient) CheckBatchAvailability(daMetaData *da.DASubmitMetaData) da.ResultCheckBatch {
func (c *DataAvailabilityLayerClient) CheckBatchAvailability(daPath string) da.ResultCheckBatch {
return da.ResultCheckBatch{
BaseResult: da.BaseResult{
Code: da.StatusSuccess,
Expand Down
4 changes: 2 additions & 2 deletions da/avail/avail_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,10 @@ func TestRetrieveBatches(t *testing.T) {
mockSubstrateApiClient.On("GetBlock", mock.Anything).Return(signedBlock, nil)

// Retrieve the batches and make sure we only get the batches relevant for our app id
daMetaData := &da.DASubmitMetaData{
daMetaData := &avail.SubmitMetaData{
Height: 1,
}
batchResult := dalc.RetrieveBatches(daMetaData)
batchResult := dalc.RetrieveBatches(daMetaData.ToPath())
assert.Equal(1, len(batchResult.Batches))
assert.Equal(batch1.StartHeight(), batchResult.Batches[0].StartHeight())
}
Loading
Loading