Skip to content

Commit

Permalink
feat: Store block timestamps.
Browse files Browse the repository at this point in the history 
I took into account backward compatibility when adding the timestamp
field into the JSON struct.

I tried to add a test to check if we are still backward compatible. It
seems that the std.Tx struct changed over time, so we won't be able to
recover previous data.

Signed-off-by: Antonio Navarro Perez <[email protected]>
  • Loading branch information
@ajnavarro
ajnavarro committed Sep 3, 2024
1 parent d0c91f5 commit b52acbc
Show file tree
Hide file tree
Showing 7 changed files with 85 additions and 27 deletions.
17 changes: 9 additions & 8 deletions backup/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ import (
"fmt"
"time"

_ "github.com/gnolang/gno/gno.land/pkg/sdk/vm"

"github.com/gnolang/tx-archive/backup/client"
"github.com/gnolang/tx-archive/backup/writer"
"github.com/gnolang/tx-archive/log"
"github.com/gnolang/tx-archive/log/noop"
"github.com/gnolang/tx-archive/types"

_ "github.com/gnolang/gno/gno.land/pkg/sdk/vm"
)

// Service is the chain backup service
Expand Down Expand Up @@ -56,22 +56,23 @@ func (s *Service) ExecuteBackup(ctx context.Context, cfg Config) error {
// Keep track of total txs backed up
totalTxs := uint64(0)

fetchAndWrite := func(block uint64) error {
txs, txErr := s.client.GetBlockTransactions(block)
fetchAndWrite := func(height uint64) error {
block, txErr := s.client.GetBlockTransactions(height)
if txErr != nil {
return fmt.Errorf("unable to fetch block transactions, %w", txErr)
}

// Skip empty blocks
if len(txs) == 0 {
if len(block.Txs) == 0 {
return nil
}

// Save the block transaction data, if any
for _, tx := range txs {
for _, tx := range block.Txs {
data := &types.TxData{
Tx: tx,
BlockNum: block,
Tx: tx,
BlockNum: block.Height,
Timestamp: block.Timestamp,
}

// Write the tx data to the file
Expand Down
30 changes: 23 additions & 7 deletions backup/backup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ import (

"github.com/gnolang/gno/tm2/pkg/amino"
"github.com/gnolang/gno/tm2/pkg/std"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/gnolang/tx-archive/backup/client"
"github.com/gnolang/tx-archive/backup/writer/standard"
"github.com/gnolang/tx-archive/log/noop"
"github.com/gnolang/tx-archive/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestBackup_DetermineRightBound(t *testing.T) {
Expand Down Expand Up @@ -96,17 +98,23 @@ func TestBackup_ExecuteBackup_FixedRange(t *testing.T) {

cfg = DefaultConfig()

blockTime = time.Date(1987, 06, 24, 6, 32, 11, 0, time.FixedZone("Europe/Madrid", 0))

Check failure on line 101 in backup/backup_test.go

View workflow job for this annotation

GitHub Actions / Go Linter / lint 

octalLiteral: use new octal literal style, 0o6 (gocritic)

mockClient = &mockClient{
getLatestBlockNumberFn: func() (uint64, error) {
return toBlock, nil
},
getBlockTransactionsFn: func(blockNum uint64) ([]std.Tx, error) {
getBlockTransactionsFn: func(blockNum uint64) (*client.Block, error) {
// Sanity check
if blockNum < fromBlock && blockNum > toBlock {
t.Fatal("invalid block number requested")
}

return []std.Tx{exampleTx}, nil // 1 tx per block
return &client.Block{
Height: blockNum,
Timestamp: blockTime.Add(time.Duration(blockNum) * time.Minute).UnixMilli(),
Txs: []std.Tx{exampleTx},
}, nil // 1 tx per block
},
}
)
Expand Down Expand Up @@ -149,9 +157,10 @@ func TestBackup_ExecuteBackup_FixedRange(t *testing.T) {
if err := amino.UnmarshalJSON(scanner.Bytes(), &txData); err != nil {
t.Fatalf("unable to unmarshal JSON line, %v", err)
}

blockTime.Unix()

Check failure on line 160 in backup/backup_test.go

View workflow job for this annotation

GitHub Actions / Go Linter / lint 

expressions should not be cuddled with blocks (wsl)
assert.Equal(t, expectedBlock, txData.BlockNum)
assert.Equal(t, exampleTx, txData.Tx)
assert.Equal(t, blockTime.Add(time.Duration(expectedBlock)*time.Minute).Local(), time.UnixMilli(txData.Timestamp))

expectedBlock++
}
Expand Down Expand Up @@ -183,11 +192,13 @@ func TestBackup_ExecuteBackup_Watch(t *testing.T) {

cfg = DefaultConfig()

blockTime = time.Date(1987, 06, 24, 6, 32, 11, 0, time.FixedZone("Europe/Madrid", 0))

Check failure on line 195 in backup/backup_test.go

View workflow job for this annotation

GitHub Actions / Go Linter / lint 

octalLiteral: use new octal literal style, 0o6 (gocritic)

mockClient = &mockClient{
getLatestBlockNumberFn: func() (uint64, error) {
return toBlock, nil
},
getBlockTransactionsFn: func(blockNum uint64) ([]std.Tx, error) {
getBlockTransactionsFn: func(blockNum uint64) (*client.Block, error) {
// Sanity check
if blockNum < fromBlock && blockNum > toBlock {
t.Fatal("invalid block number requested")
Expand All @@ -198,7 +209,11 @@ func TestBackup_ExecuteBackup_Watch(t *testing.T) {
cancelFn()
}

return []std.Tx{exampleTx}, nil // 1 tx per block
return &client.Block{
Height: blockNum,
Timestamp: blockTime.Add(time.Duration(blockNum) * time.Minute).UnixMilli(),
Txs: []std.Tx{exampleTx},
}, nil // 1 tx per block
},
}
)
Expand Down Expand Up @@ -246,6 +261,7 @@ func TestBackup_ExecuteBackup_Watch(t *testing.T) {

assert.Equal(t, expectedBlock, txData.BlockNum)
assert.Equal(t, exampleTx, txData.Tx)
assert.Equal(t, blockTime.Add(time.Duration(expectedBlock)*time.Minute).Local(), time.UnixMilli(txData.Timestamp))

expectedBlock++
}
Expand Down
12 changes: 10 additions & 2 deletions backup/client/client.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package client

import "github.com/gnolang/gno/tm2/pkg/std"
import (
"github.com/gnolang/gno/tm2/pkg/std"
)

// Client defines the client interface for fetching chain data
type Client interface {
Expand All @@ -9,5 +11,11 @@ type Client interface {

// GetBlockTransactions returns the transactions contained
// within the specified block, if any
GetBlockTransactions(uint64) ([]std.Tx, error)
GetBlockTransactions(uint64) (*Block, error)
}

type Block struct {

Check failure on line 17 in backup/client/client.go

View workflow job for this annotation

GitHub Actions / Go Linter / lint 

fieldalignment: struct with 24 pointer bytes could be 8 (govet)
Height uint64
Timestamp int64
Txs []std.Tx
}
13 changes: 10 additions & 3 deletions backup/client/http/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,16 @@ package http
import (
"fmt"

_ "github.com/gnolang/gno/gno.land/pkg/sdk/vm"
"github.com/gnolang/gno/tm2/pkg/amino"
rpcClient "github.com/gnolang/gno/tm2/pkg/bft/rpc/client"
"github.com/gnolang/gno/tm2/pkg/std"

_ "github.com/gnolang/gno/gno.land/pkg/sdk/vm"
"github.com/gnolang/tx-archive/backup/client"
)

var _ client.Client = &Client{}

// Client is the TM2 HTTP client
type Client struct {
client rpcClient.Client
Expand Down Expand Up @@ -40,7 +43,7 @@ func (c *Client) GetLatestBlockNumber() (uint64, error) {
return uint64(status.SyncInfo.LatestBlockHeight), nil
}

func (c *Client) GetBlockTransactions(blockNum uint64) ([]std.Tx, error) {
func (c *Client) GetBlockTransactions(blockNum uint64) (*client.Block, error) {
// Fetch the block
blockNumInt64 := int64(blockNum)

Expand Down Expand Up @@ -68,5 +71,9 @@ func (c *Client) GetBlockTransactions(blockNum uint64) ([]std.Tx, error) {
txs = append(txs, tx)
}

return txs, nil
return &client.Block{
Timestamp: block.Block.Time.UnixMilli(),
Height: blockNum,
Txs: txs,
}, nil
}
8 changes: 5 additions & 3 deletions backup/mock_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package backup

import "github.com/gnolang/gno/tm2/pkg/std"
import (
"github.com/gnolang/tx-archive/backup/client"
)

type (
getLatestBlockNumberDelegate func() (uint64, error)
getBlockTransactionsDelegate func(uint64) ([]std.Tx, error)
getBlockTransactionsDelegate func(uint64) (*client.Block, error)
)

type mockClient struct {
Expand All @@ -20,7 +22,7 @@ func (m *mockClient) GetLatestBlockNumber() (uint64, error) {
return 0, nil
}

func (m *mockClient) GetBlockTransactions(blockNum uint64) ([]std.Tx, error) {
func (m *mockClient) GetBlockTransactions(blockNum uint64) (*client.Block, error) {
if m.getBlockTransactionsFn != nil {
return m.getBlockTransactionsFn(blockNum)
}
Expand Down
23 changes: 22 additions & 1 deletion restore/restore_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,13 @@ import (
"testing"
"time"

"github.com/gnolang/gno/tm2/pkg/amino"
"github.com/gnolang/gno/tm2/pkg/std"
"github.com/gnolang/tx-archive/log/noop"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/gnolang/tx-archive/log/noop"
"github.com/gnolang/tx-archive/types"
)

func TestRestore_ExecuteRestore(t *testing.T) {
Expand Down Expand Up @@ -145,3 +149,20 @@ func TestRestore_ExecuteRestore_Watch(t *testing.T) {
assert.Equal(t, exampleTx, tx)
}
}

func TestRestore_BackwardCompatible(t *testing.T) {
t.Skip("WORTH IT?")

t.Parallel()

var oldTx = `{"tx":{"msg":[{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path":"gno.land/r/demo/wugnot","func":"Approve","args":["g126swhfaq2vyvvjywevhgw7lv9hg8qan93dasu8","18446744073709551615"]},{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path":"gno.land/r/gnoswap/v2/gns","func":"Approve","args":["g126swhfaq2vyvvjywevhgw7lv9hg8qan93dasu8","18446744073709551615"]},{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path":"gno.land/r/demo/wugnot","func":"Approve","args":["g14fclvfqynndp0l6kpyxkpgn4sljw9rr96hz46l","18446744073709551615"]},{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path":"gno.land/r/gnoswap/v2/position","func":"CollectFee","args":["26"]},{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path":"gno.land/r/gnoswap/v2/staker","func":"CollectReward","args":["26","true"]},{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path":"gno.land/r/demo/wugnot","func":"Approve","args":["g126swhfaq2vyvvjywevhgw7lv9hg8qan93dasu8","18446744073709551615"]},{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path":"gno.land/r/gnoswap/v2/gns","func":"Approve","args":["g126swhfaq2vyvvjywevhgw7lv9hg8qan93dasu8","18446744073709551615"]},{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path":"gno.land/r/gnoswap/v2/position","func":"CollectFee","args":["146"]}],"fee":{"gas_wanted":"100000000","gas_fee":"1ugnot"},"signatures":[{"pub_key":{"@type":"/tm.PubKeySecp256k1","value":"Atgv/+TCwlR+jzjx94p4Ik0IuGET4J/q2q9ciaL4UOQh"},"signature":"iVfxsF37nRtgqyq9tMRMhyFLxp5RVdpI1r0mSHLmdg5aly0w82/in0ECey2PSpRk2UQ/fCtMpyOzaqIXiVKC4Q=="}],"memo":""},"blockNum":"1194460"}`

var out types.TxData
err := amino.UnmarshalJSON([]byte(oldTx), &out)
require.NoError(t, err)

require.Zero(t, out.Timestamp)
// require.Equal(t, 23, out.BlockNum)

Check failure on line 165 in restore/restore_test.go

View workflow job for this annotation

GitHub Actions / Go Linter / lint 

commentedOutCode: may want to remove commented-out code (gocritic)
// require.Equal(t, 23, out.Tx.Memo)
// require.Equal(t, 23, out.Tx.Msgs)
}

Check failure on line 168 in restore/restore_test.go

View workflow job for this annotation

GitHub Actions / Go Linter / lint 

block should not end with a whitespace (or comment) (wsl)
9 changes: 6 additions & 3 deletions types/types.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
package types

import "github.com/gnolang/gno/tm2/pkg/std"
import (
"github.com/gnolang/gno/tm2/pkg/std"
)

// TxData contains the single block transaction,
// along with the block information
type TxData struct {
Tx std.Tx `json:"tx"`
BlockNum uint64 `json:"blockNum"`
Tx std.Tx `json:"tx"`
BlockNum uint64 `json:"blockNum"`
Timestamp int64 `json:"bt"`

Check failure on line 12 in types/types.go

View workflow job for this annotation

GitHub Actions / Go Linter / lint 

json(goCamel): got 'bt' want 'timestamp' (tagliatelle)
}

0 comments on commit b52acbc

Please sign in to comment.