merge master

CODEOWNERS

@@ -23,11 +23,11 @@
 /module/chunking/** @ramtinms
 /integration/tests/verification @ramtinms @yhassanzadeh13
 
-# Ledger Stream 
+# Ledger Stream
 /ledger/** @ramtinms @AlexHentschel
 
-# FVM Stream 
-/fvm/**  @ramtinms @janezpodhostnik @pattyshack
+# FVM Stream
+/fvm/**  @ramtinms @janezpodhostnik
 
 # Networking Stream
 /network/** @yhassanzadeh13
@@ -45,10 +45,7 @@
 /tools/test_monitor/** @gomisha
 
 # Performance Stream
-/integration/benchmark/** @SaveTheRbtz @gomisha
-/integration/localnet/** @SaveTheRbtz
-/module/profiler/** @SaveTheRbtz @pattyshack
-/module/tracer.go @SaveTheRbtz @pattyshack
+/integration/benchmark/** @gomisha
 
 # Execution Sync
 /module/executiondatasync/** @peterargue

Makefile

@@ -77,7 +77,7 @@ install-tools: crypto_setup_gopath check-go-version install-mock-generators
 	go install golang.org/x/tools/cmd/stringer@master;
 
 .PHONY: verify-mocks
-verify-mocks: generate-mocks
+verify-mocks: tidy generate-mocks
 	git diff --exit-code
 
 ############################################################################################
@@ -194,7 +194,6 @@ generate-mocks: install-mock-generators
 	mockery --name 'API' --dir="./engine/protocol" --case=underscore --output="./engine/protocol/mock" --outpkg="mock"
 	mockery --name '.*' --dir="./engine/access/state_stream" --case=underscore --output="./engine/access/state_stream/mock" --outpkg="mock"
 	mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/backend" --case=underscore --output="./engine/access/rpc/backend/mock" --outpkg="mock"
-	mockery --name 'IngestRPC' --dir="./engine/execution/ingestion" --case=underscore --tags relic --output="./engine/execution/ingestion/mock" --outpkg="mock"
 	mockery --name '.*' --dir=model/fingerprint --case=underscore --output="./model/fingerprint/mock" --outpkg="mock"
 	mockery --name 'ExecForkActor' --structname 'ExecForkActorMock' --dir=module/mempool/consensus/mock/ --case=underscore --output="./module/mempool/consensus/mock/" --outpkg="mock"
 	mockery --name '.*' --dir=engine/verification/fetcher/ --case=underscore --output="./engine/verification/fetcher/mock" --outpkg="mockfetcher"
@@ -534,7 +533,7 @@ docker-push-access-without-netgo:
 .PHONY: docker-push-access-latest
 docker-push-access-latest: docker-push-access
 	docker push "$(CONTAINER_REGISTRY)/access:latest"
-	
+
 
 .PHONY: docker-push-observer
 docker-push-observer:

admin/README.md

@@ -46,6 +46,17 @@ curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"
 curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "get-transactions", "data": { "start-height": 340, "end-height": 343 }}'
 ```
 
+### To get blocks for ranges (works for any node type, for block payload, only prints the collection ids)
+```
+curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "read-range-blocks", "data": { "start-height": 105172044, "end-height": 105172047 }}'
+```
+
+### To get cluster block for ranges (only available to collection nodes, only prints the transaction ids)
+
+```
+curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "read-range-cluster-blocks", "data": { "chain-id": "cluster-576-e8af4702d837acb77868a95f61eb212f90b14c6b7d61c89f48949fd27d1a269b", "start-height": 25077, "end-height": 25080 }}'
+```
+
 ### To get execution data for a block by execution_data_id (only available execution nodes and access nodes with execution sync enabled)
 ```
 curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "read-execution-data", "data": { "execution_data_id": "2fff2b05e7226c58e3c14b3549ab44a354754761c5baa721ea0d1ea26d069dc4" }}'

admin/commands/execution/stop_at_height_test.go

@@ -3,11 +3,13 @@ package execution
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 
 	"github.com/onflow/flow-go/admin"
+	"github.com/onflow/flow-go/engine"
 	"github.com/onflow/flow-go/engine/execution/ingestion/stop"
 	"github.com/onflow/flow-go/model/flow"
 )
@@ -90,6 +92,8 @@ func TestCommandParsing(t *testing.T) {
 func TestCommandsSetsValues(t *testing.T) {
 
 	stopControl := stop.NewStopControl(
+		engine.NewUnit(),
+		time.Second,
 		zerolog.Nop(),
 		nil,
 		nil,

admin/commands/storage/helper.go

@@ -5,6 +5,7 @@ import (
 	"math"
 	"strings"
 
+	"github.com/onflow/flow-go/admin"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/state/protocol"
 )
@@ -90,3 +91,69 @@ func getBlockHeader(state protocol.State, req *blocksRequest) (*flow.Header, err
 		return nil, fmt.Errorf("invalid request type: %v", req.requestType)
 	}
 }
+
+func parseHeightRangeRequestData(req *admin.CommandRequest) (*heightRangeReqData, error) {
+	input, ok := req.Data.(map[string]interface{})
+	if !ok {
+		return nil, admin.NewInvalidAdminReqFormatError("missing 'data' field")
+	}
+
+	startHeight, err := findUint64(input, "start-height")
+	if err != nil {
+		return nil, fmt.Errorf("invalid start-height: %w", err)
+	}
+
+	endHeight, err := findUint64(input, "end-height")
+	if err != nil {
+		return nil, fmt.Errorf("invalid end-height: %w", err)
+	}
+
+	if endHeight < startHeight {
+		return nil, admin.NewInvalidAdminReqErrorf("end-height %v should not be smaller than start-height %v", endHeight, startHeight)
+	}
+
+	return &heightRangeReqData{
+		startHeight: startHeight,
+		endHeight:   endHeight,
+	}, nil
+}
+
+func parseString(req *admin.CommandRequest, field string) (string, error) {
+	input, ok := req.Data.(map[string]interface{})
+	if !ok {
+		return "", admin.NewInvalidAdminReqFormatError("missing 'data' field")
+	}
+	fieldValue, err := findString(input, field)
+	if err != nil {
+		return "", admin.NewInvalidAdminReqErrorf("missing %v field", field)
+	}
+	return fieldValue, nil
+}
+
+// Returns admin.InvalidAdminReqError for invalid inputs
+func findUint64(input map[string]interface{}, field string) (uint64, error) {
+	data, ok := input[field]
+	if !ok {
+		return 0, admin.NewInvalidAdminReqErrorf("missing required field '%s'", field)
+	}
+	val, err := parseN(data)
+	if err != nil {
+		return 0, admin.NewInvalidAdminReqErrorf("invalid 'n' field: %w", err)
+	}
+
+	return uint64(val), nil
+}
+
+func findString(input map[string]interface{}, field string) (string, error) {
+	data, ok := input[field]
+	if !ok {
+		return "", admin.NewInvalidAdminReqErrorf("missing required field '%s'", field)
+	}
+
+	str, ok := data.(string)
+	if !ok {
+		return "", admin.NewInvalidAdminReqErrorf("field '%s' is not string", field)
+	}
+
+	return strings.ToLower(strings.TrimSpace(str)), nil
+}

admin/commands/storage/read_blocks.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/rs/zerolog/log"
+
 	"github.com/onflow/flow-go/admin"
 	"github.com/onflow/flow-go/admin/commands"
 	"github.com/onflow/flow-go/model/flow"
@@ -28,6 +30,8 @@ func (r *ReadBlocksCommand) Handler(ctx context.Context, req *admin.CommandReque
 	var result []*flow.Block
 	var blockID flow.Identifier
 
+	log.Info().Str("module", "admin-tool").Msgf("read blocks, data: %v", data)
+
 	if header, err := getBlockHeader(r.state, data.blocksRequest); err != nil {
 		return nil, fmt.Errorf("failed to get block header: %w", err)
 	} else {

admin/commands/storage/read_range_blocks.go

@@ -0,0 +1,51 @@
+package storage
+
+import (
+	"context"
+
+	"github.com/rs/zerolog/log"
+
+	"github.com/onflow/flow-go/admin"
+	"github.com/onflow/flow-go/admin/commands"
+	"github.com/onflow/flow-go/cmd/util/cmd/read-light-block"
+	"github.com/onflow/flow-go/storage"
+)
+
+var _ commands.AdminCommand = (*ReadRangeBlocksCommand)(nil)
+
+// 10001 instead of 10000, because 10000 won't allow a range from 10000 to 20000,
+// which is easier to type than [10001, 20000]
+const Max_Range_Block_Limit = uint64(10001)
+
+type ReadRangeBlocksCommand struct {
+	blocks storage.Blocks
+}
+
+func NewReadRangeBlocksCommand(blocks storage.Blocks) commands.AdminCommand {
+	return &ReadRangeBlocksCommand{
+		blocks: blocks,
+	}
+}
+
+func (c *ReadRangeBlocksCommand) Handler(ctx context.Context, req *admin.CommandRequest) (interface{}, error) {
+	reqData, err := parseHeightRangeRequestData(req)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Info().Str("module", "admin-tool").Msgf("read range blocks, data: %v", reqData)
+
+	if reqData.Range() > Max_Range_Block_Limit {
+		return nil, admin.NewInvalidAdminReqErrorf("getting for more than %v blocks at a time might have an impact to node's performance and is not allowed", Max_Range_Block_Limit)
+	}
+
+	lights, err := read.ReadLightBlockByHeightRange(c.blocks, reqData.startHeight, reqData.endHeight)
+	if err != nil {
+		return nil, err
+	}
+	return commands.ConvertToInterfaceList(lights)
+}
+
+func (c *ReadRangeBlocksCommand) Validator(req *admin.CommandRequest) error {
+	return nil
+}

admin/commands/storage/read_range_cluster_blocks.go

@@ -0,0 +1,67 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/dgraph-io/badger/v2"
+	"github.com/rs/zerolog/log"
+
+	"github.com/onflow/flow-go/admin"
+	"github.com/onflow/flow-go/admin/commands"
+	"github.com/onflow/flow-go/cmd/util/cmd/read-light-block"
+	"github.com/onflow/flow-go/model/flow"
+	storage "github.com/onflow/flow-go/storage/badger"
+)
+
+var _ commands.AdminCommand = (*ReadRangeClusterBlocksCommand)(nil)
+
+// 10001 instead of 10000, because 10000 won't allow a range from 10000 to 20000,
+// which is easier to type than [10001, 20000]
+const Max_Range_Cluster_Block_Limit = uint64(10001)
+
+type ReadRangeClusterBlocksCommand struct {
+	db       *badger.DB
+	headers  *storage.Headers
+	payloads *storage.ClusterPayloads
+}
+
+func NewReadRangeClusterBlocksCommand(db *badger.DB, headers *storage.Headers, payloads *storage.ClusterPayloads) commands.AdminCommand {
+	return &ReadRangeClusterBlocksCommand{
+		db:       db,
+		headers:  headers,
+		payloads: payloads,
+	}
+}
+
+func (c *ReadRangeClusterBlocksCommand) Handler(ctx context.Context, req *admin.CommandRequest) (interface{}, error) {
+	chainID, err := parseString(req, "chain-id")
+	if err != nil {
+		return nil, err
+	}
+
+	reqData, err := parseHeightRangeRequestData(req)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Info().Str("module", "admin-tool").Msgf("read range cluster blocks, data: %v", reqData)
+
+	if reqData.Range() > Max_Range_Cluster_Block_Limit {
+		return nil, admin.NewInvalidAdminReqErrorf("getting for more than %v blocks at a time might have an impact to node's performance and is not allowed", Max_Range_Cluster_Block_Limit)
+	}
+
+	clusterBlocks := storage.NewClusterBlocks(
+		c.db, flow.ChainID(chainID), c.headers, c.payloads,
+	)
+
+	lights, err := read.ReadClusterLightBlockByHeightRange(clusterBlocks, reqData.startHeight, reqData.endHeight)
+	if err != nil {
+		return nil, fmt.Errorf("could not get with chainID id %v: %w", chainID, err)
+	}
+	return commands.ConvertToInterfaceList(lights)
+}
+
+func (c *ReadRangeClusterBlocksCommand) Validator(req *admin.CommandRequest) error {
+	return nil
+}

admin/commands/storage/read_transactions.go

@@ -14,14 +14,15 @@ import (
 
 var _ commands.AdminCommand = (*GetTransactionsCommand)(nil)
 
-// max number of block height to query transactions from
-var MAX_HEIGHT_RANGE = uint64(1000)
-
-type getTransactionsReqData struct {
+type heightRangeReqData struct {
 	startHeight uint64
 	endHeight   uint64
 }
 
+func (d heightRangeReqData) Range() uint64 {
+	return d.endHeight - d.startHeight + 1
+}
+
 type GetTransactionsCommand struct {
 	state       protocol.State
 	payloads    storage.Payloads
@@ -37,7 +38,12 @@ func NewGetTransactionsCommand(state protocol.State, payloads storage.Payloads,
 }
 
 func (c *GetTransactionsCommand) Handler(ctx context.Context, req *admin.CommandRequest) (interface{}, error) {
-	data := req.ValidatorData.(*getTransactionsReqData)
+	data := req.ValidatorData.(*heightRangeReqData)
+
+	limit := uint64(10001)
+	if data.Range() > limit {
+		return nil, admin.NewInvalidAdminReqErrorf("getting transactions for more than %v blocks at a time might have an impact to node's performance and is not allowed", limit)
+	}
 
 	finder := &transactions.Finder{
 		State:       c.state,
@@ -55,50 +61,13 @@ func (c *GetTransactionsCommand) Handler(ctx context.Context, req *admin.Command
 	return commands.ConvertToInterfaceList(blocks)
 }
 
-// Returns admin.InvalidAdminReqError for invalid inputs
-func findUint64(input map[string]interface{}, field string) (uint64, error) {
-	data, ok := input[field]
-	if !ok {
-		return 0, admin.NewInvalidAdminReqErrorf("missing required field '%s'", field)
-	}
-	val, err := parseN(data)
-	if err != nil {
-		return 0, admin.NewInvalidAdminReqErrorf("invalid 'n' field: %w", err)
-	}
-
-	return uint64(val), nil
-}
-
 // Validator validates the request.
 // Returns admin.InvalidAdminReqError for invalid/malformed requests.
 func (c *GetTransactionsCommand) Validator(req *admin.CommandRequest) error {
-	input, ok := req.Data.(map[string]interface{})
-	if !ok {
-		return admin.NewInvalidAdminReqFormatError("expected map[string]any")
-	}
-
-	startHeight, err := findUint64(input, "start-height")
-	if err != nil {
-		return err
-	}
-
-	endHeight, err := findUint64(input, "end-height")
+	data, err := parseHeightRangeRequestData(req)
 	if err != nil {
 		return err
 	}
-
-	if endHeight < startHeight {
-		return admin.NewInvalidAdminReqErrorf("endHeight %v should not be smaller than startHeight %v", endHeight, startHeight)
-	}
-
-	if endHeight-startHeight+1 > MAX_HEIGHT_RANGE {
-		return admin.NewInvalidAdminReqErrorf("getting transactions for more than %v blocks at a time might have an impact to node's performance and is not allowed", MAX_HEIGHT_RANGE)
-	}
-
-	req.ValidatorData = &getTransactionsReqData{
-		startHeight: startHeight,
-		endHeight:   endHeight,
-	}
-
+	req.ValidatorData = data
 	return nil
 }